New project structure

Beta of extract.js for extracting english locale

server/.eslintrc

@@ -0,0 +1,3 @@
+{
+    "extends": "nodemailer"
+}

server/.gitignore

@@ -0,0 +1,7 @@
+/config/development.*
+/config/production.*
+/config/test.*
+/workers/reports/config/development.*
+/workers/reports/config/production.*
+/workers/reports/config/test.*
+/files

server/Gruntfile.js

@@ -0,0 +1,29 @@
+'use strict';
+
+module.exports = function (grunt) {
+
+    // Project configuration.
+    grunt.initConfig({
+        eslint: {
+            all: [
+                'lib/**/*.js',
+                'models/**/*.js',
+                'routes/**/*.js',
+                'services/**/*.js',
+                'lib/**/*.js',
+                'test/**/*.js',
+                'workers/**/*.js',
+                'app-builder.js',
+                'index.js',
+                'Gruntfile.js',
+                ]
+        }
+    });
+
+    // Load the plugin(s)
+    grunt.loadNpmTasks('grunt-eslint');
+    grunt.task.loadTasks('tasks');
+
+    // Tasks
+    grunt.registerTask('default', ['eslint']);
+};

server/app-builder.js

@@ -0,0 +1,371 @@
+'use strict';
+
+const config = require('config');
+const log = require('./lib/log');
+
+const express = require('express');
+const expressLocale = require('express-locale');
+const bodyParser = require('body-parser');
+const path = require('path');
+const favicon = require('serve-favicon');
+const logger = require('morgan');
+const cookieParser = require('cookie-parser');
+const session = require('express-session');
+const flash = require('connect-flash');
+const hbs = require('hbs');
+const compression = require('compression');
+const passport = require('./lib/passport');
+const contextHelpers = require('./lib/context-helpers');
+
+const api = require('./routes/api');
+
+// These are routes for the new React-based client
+const reports = require('./routes/reports');
+const subscriptions = require('./routes/subscriptions');
+const subscription = require('./routes/subscription');
+const sandboxedMosaico = require('./routes/sandboxed-mosaico');
+const sandboxedCKEditor = require('./routes/sandboxed-ckeditor');
+const sandboxedGrapesJS = require('./routes/sandboxed-grapesjs');
+const sandboxedCodeEditor = require('./routes/sandboxed-codeeditor');
+const files = require('./routes/files');
+const links = require('./routes/links');
+const archive = require('./routes/archive');
+const webhooks = require('./routes/webhooks');
+
+const namespacesRest = require('./routes/rest/namespaces');
+const sendConfigurationsRest = require('./routes/rest/send-configurations');
+const usersRest = require('./routes/rest/users');
+const accountRest = require('./routes/rest/account');
+const reportTemplatesRest = require('./routes/rest/report-templates');
+const reportsRest = require('./routes/rest/reports');
+const campaignsRest = require('./routes/rest/campaigns');
+const triggersRest = require('./routes/rest/triggers');
+const listsRest = require('./routes/rest/lists');
+const formsRest = require('./routes/rest/forms');
+const fieldsRest = require('./routes/rest/fields');
+const importsRest = require('./routes/rest/imports');
+const importRunsRest = require('./routes/rest/import-runs');
+const sharesRest = require('./routes/rest/shares');
+const segmentsRest = require('./routes/rest/segments');
+const subscriptionsRest = require('./routes/rest/subscriptions');
+const templatesRest = require('./routes/rest/templates');
+const mosaicoTemplatesRest = require('./routes/rest/mosaico-templates');
+const blacklistRest = require('./routes/rest/blacklist');
+const editorsRest = require('./routes/rest/editors');
+const filesRest = require('./routes/rest/files');
+const settingsRest = require('./routes/rest/settings');
+
+const index = require('./routes/index');
+
+const interoperableErrors = require('../shared/interoperable-errors');
+
+const { getTrustedUrl } = require('./lib/urls');
+const { AppType } = require('../shared/app');
+
+hbs.registerPartials(__dirname + '/views/partials');
+hbs.registerPartials(__dirname + '/views/subscription/partials/');
+
+/**
+ * We need this helper to make sure that we consume flash messages only
+ * when we are able to actually display these. Otherwise we might end up
+ * in a situation where we consume a flash messages but then comes a redirect
+ * and the message is never displayed
+ */
+hbs.registerHelper('flash_messages', function () { // eslint-disable-line prefer-arrow-callback
+    if (typeof this.flash !== 'function') { // eslint-disable-line no-invalid-this
+        return '';
+    }
+
+    const messages = this.flash(); // eslint-disable-line no-invalid-this
+    const response = [];
+
+    // group messages by type
+    for (const key in messages) {
+        let el = '<div class="alert alert-' + key + ' alert-dismissible" role="alert"><button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">&times;</span></button>';
+
+        if (key === 'danger') {
+            el += '<span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> ';
+        }
+
+        let rows = [];
+
+        for (const message of messages[key]) {
+            rows.push(hbs.handlebars.escapeExpression(message).replace(/(\r\n|\n|\r)/gm, '<br>'));
+        }
+
+        if (rows.length > 1) {
+            el += '<p>' + rows.join('</p>\n<p>') + '</p>';
+        } else {
+            el += rows.join('');
+        }
+
+        el += '</div>';
+
+        response.push(el);
+    }
+
+    return new hbs.handlebars.SafeString(
+        response.join('\n')
+    );
+});
+
+
+
+function createApp(appType) {
+    const app = express();
+
+    function install404Fallback(url) {
+        app.use(url, (req, res, next) => {
+            next(new interoperableErrors.NotFoundError());
+        });
+
+        app.use(url + '/*', (req, res, next) => {
+            next(new interoperableErrors.NotFoundError());
+        });
+    }
+
+    function useWith404Fallback(url, route) {
+        app.use(url, route);
+        install404Fallback(url);
+    }
+
+    // view engine setup
+    app.set('views', path.join(__dirname, 'views'));
+    app.set('view engine', 'hbs');
+
+    // Handle proxies. Needed to resolve client IP
+    if (config.www.proxy) {
+        app.set('trust proxy', config.www.proxy);
+    }
+
+    // Do not expose software used
+    app.disable('x-powered-by');
+
+    app.use(compression());
+    app.use(favicon(path.join(__dirname, 'client', 'static', 'favicon.ico')));
+
+    app.use(logger(config.www.log, {
+        stream: {
+            write: message => {
+                message = (message || '').toString();
+                if (message) {
+                    log.info('HTTP', message.replace('\n', '').trim());
+                }
+            }
+        }
+    }));
+
+    app.use(cookieParser());
+    app.use(session({
+        secret: config.www.secret,
+        saveUninitialized: false,
+        resave: false
+    }));
+
+    app.use(expressLocale({
+        priority: ['query', 'accept-language', 'default'],
+        query: {
+            name: 'language'
+        },
+        default: 'en_US'
+    }));
+
+    app.use(flash());
+
+    app.use(bodyParser.urlencoded({
+        extended: true,
+        limit: config.www.postSize
+    }));
+
+    app.use(bodyParser.text({
+        limit: config.www.postSize
+    }));
+
+    app.use(bodyParser.json({
+        limit: config.www.postSize
+    }));
+
+    if (appType === AppType.TRUSTED) {
+        passport.setupRegularAuth(app);
+    } else if (appType === AppType.SANDBOXED) {
+        app.use(passport.tryAuthByRestrictedAccessToken);
+    }
+
+    useWith404Fallback('/static', express.static(path.join(__dirname, 'client', 'static')));
+    useWith404Fallback('/mailtrain', express.static(path.join(__dirname, 'client', 'dist')));
+    useWith404Fallback('/locales', express.static(path.join(__dirname, 'client', 'locales')));
+
+
+    // Make sure flash messages are available
+    // Currently, flash messages are used only from routes/subscription.js
+    app.use((req, res, next) => {
+        res.locals.flash = req.flash.bind(req);
+        next();
+    });
+
+    // Endpoint under /api are authenticated by access token
+    app.all('/api/*', passport.authByAccessToken);
+
+    // Marks the following endpoint to return JSON object when error occurs
+    app.all('/api/*', (req, res, next) => {
+        req.needsAPIJSONResponse = true;
+        next();
+    });
+
+    app.all('/rest/*', (req, res, next) => {
+        req.needsRESTJSONResponse = true;
+        next();
+    });
+
+    // Initializes the request context to be used for authorization
+    app.use((req, res, next) => {
+        req.context = contextHelpers.getRequestContext(req);
+        next();
+    });
+
+    if (appType === AppType.PUBLIC) {
+        useWith404Fallback('/subscription', subscription);
+        useWith404Fallback('/links', links);
+        useWith404Fallback('/archive', archive);
+        useWith404Fallback('/files', files);
+    }
+
+    useWith404Fallback('/mosaico', sandboxedMosaico.getRouter(appType));
+    useWith404Fallback('/ckeditor', sandboxedCKEditor.getRouter(appType));
+    useWith404Fallback('/grapesjs', sandboxedGrapesJS.getRouter(appType));
+    useWith404Fallback('/codeeditor', sandboxedCodeEditor.getRouter(appType));
+
+    if (appType === AppType.TRUSTED || appType === AppType.SANDBOXED) {
+        if (config.reports && config.reports.enabled === true) {
+            useWith404Fallback('/reports', reports);
+        }
+
+        useWith404Fallback('/subscriptions', subscriptions);
+        useWith404Fallback('/webhooks', webhooks);
+
+        // API endpoints
+        useWith404Fallback('/api', api);
+
+        // REST endpoints
+        app.use('/rest', namespacesRest);
+        app.use('/rest', sendConfigurationsRest);
+        app.use('/rest', usersRest);
+        app.use('/rest', accountRest);
+        app.use('/rest', campaignsRest);
+        app.use('/rest', triggersRest);
+        app.use('/rest', listsRest);
+        app.use('/rest', formsRest);
+        app.use('/rest', fieldsRest);
+        app.use('/rest', importsRest);
+        app.use('/rest', importRunsRest);
+        app.use('/rest', sharesRest);
+        app.use('/rest', segmentsRest);
+        app.use('/rest', subscriptionsRest);
+        app.use('/rest', templatesRest);
+        app.use('/rest', mosaicoTemplatesRest);
+        app.use('/rest', blacklistRest);
+        app.use('/rest', editorsRest);
+        app.use('/rest', filesRest);
+        app.use('/rest', settingsRest);
+
+        if (config.reports && config.reports.enabled === true) {
+            app.use('/rest', reportTemplatesRest);
+            app.use('/rest', reportsRest);
+        }
+        install404Fallback('/rest');
+    }
+
+    app.use('/', index.getRouter(appType));
+
+    // Error handlers
+    if (app.get('env') === 'development' || app.get('env') === 'test') {
+        // development error handler
+        // will print stacktrace
+        app.use((err, req, res, next) => {
+            if (!err) {
+                return next();
+            }
+
+            if (req.needsRESTJSONResponse) {
+                const resp = {
+                    message: err.message,
+                    error: err
+                };
+
+                if (err instanceof interoperableErrors.InteroperableError) {
+                    resp.type = err.type;
+                    resp.data = err.data;
+                }
+
+                res.status(err.status || 500).json(resp);
+
+            } else if (req.needsAPIJSONResponse) {
+                const resp = {
+                    error: err.message || err,
+                    data: []
+                };
+
+                return status(err.status || 500).json(resp);
+
+            } else {
+                if (err instanceof interoperableErrors.NotLoggedInError) {
+                    return res.redirect(getTrustedUrl('/account/login?next=' + encodeURIComponent(req.originalUrl)));
+                } else {
+                    res.status(err.status || 500);
+                    res.render('error', {
+                        message: err.message,
+                        error: err
+                    });
+                }
+            }
+
+        });
+    } else {
+        // production error handler
+        // no stacktraces leaked to user
+        app.use((err, req, res, next) => {
+            if (!err) {
+                return next();
+            }
+
+            if (req.needsRESTJSONResponse) {
+                const resp = {
+                    message: err.message,
+                    error: {}
+                };
+
+                if (err instanceof interoperableErrors.InteroperableError) {
+                    resp.type = err.type;
+                    resp.data = err.data;
+                }
+
+                res.status(err.status || 500).json(resp);
+
+            } else if (req.needsAPIJSONResponse) {
+                const resp = {
+                    error: err.message || err,
+                    data: []
+                };
+
+                return res.status(err.status || 500).json(resp);
+
+            } else {
+                // TODO: Render interoperable errors using a special client that does internationalization of the error message
+
+                if (err instanceof interoperableErrors.NotLoggedInError) {
+                    return res.redirect(getTrustedUrl('/account/login?next=' + encodeURIComponent(req.originalUrl)));
+                } else {
+                    res.status(err.status || 500);
+                    res.render('error', {
+                        message: err.message,
+                        error: {}
+                    });
+                }
+            }
+        });
+    }
+
+    return app;
+}
+
+module.exports.createApp = createApp;

server/config/default.yaml

@@ -0,0 +1,265 @@
+# This file is the default config file for Mailtrain. To use a environment specific
+# configuration add new file {ENV}.{ext} (eg. production.yaml) to the same folder.
+# {ENV} is defined by NODE_ENV environment variable.
+#
+# Do not modify this file directly, otherwise you might lose your modifications when upgrading
+#
+# You should only define the options you want to change in your additional config file.
+# For example if the only thing you want to change is the port number for the www server
+# then your additional config file should look like this:
+#   # production.yaml
+#   www:
+#     port: 80
+
+# Process title visible in monitoring logs and process listing
+title: mailtrain
+
+# Enabled HTML editors
+editors:
+- grapesjs
+- mosaico
+- mosaicoWithFsTemplate
+- ckeditor5
+- ckeditor4
+- codeeditor
+
+# Default language to use
+language: en
+
+# Inject custom scripts in subscription/layout.mjml.hbs
+# customSubscriptionScripts: [/custom/hello-world.js]
+
+# If you start out as a root user (eg. if you want to use ports lower than 1000)
+# then you can downgrade the user once all services are up and running
+#user: mailtrain
+#group: mailtrain
+
+# If Mailtrain is started as root, Reports feature drops the privileges of script generating the report to disallow
+# any modifications of Mailtrain code and even prohibits reading the production configuration (which contains the MySQL
+# password for read/write operations). The roUser/roGroup determines the user to be used
+#roUser: nobody
+#roGroup: nogroup
+
+log:
+  # silly|verbose|info|http|warn|error|silent
+  level: verbose
+
+www:
+  # HTTP port to listen on for trusted requests (logged-in users)
+  trustedPort: 3000
+  # HTTP port to listen on for sandboxed requests (logged-in users)
+  sandboxPort: 8081
+  # HTTP port to listen on for public requests (campaign recipients)
+  publicPort: 8082
+  # HTTP interface to listen on
+  host: 0.0.0.0
+  # URL base for trusted urls (logged-in users). It must be absolute (starting with http:// or https://). If Mailtrain is served on
+  # a non-standard port (e.g. 3000), the URL must also specify the port.
+  trustedUrlBase: http://localhost:3000
+  # URL base for sandbox urls (logged-in users). It must be absolute (starting with http:// or https://) and contain the sandbox port.
+  sandboxUrlBase: http://localhost:8081
+  # URL base for public urls (campaign recipients). It must be absolute (starting with http:// or https://) and contain the sandbox port.
+  publicUrlBase: http://localhost:8082
+
+
+  # Secret for signing the session ID cookie
+  secret: a cat
+  # Session length in seconds when remember me is checked
+  remember: 2592000 # 30 days
+  # logger interface for expressjs morgan
+  log: dev
+  # Is the server behind a proxy? true/false
+  # Set this to true if you are serving Mailtrain as a virtual domain through Nginx or Apache
+  proxy: false
+  # maximum POST body size
+  postSize: 2MB
+
+mysql:
+  host: localhost
+  user: mailtrain
+  password: mailtrain
+  database: mailtrain
+  # Some installations, eg. MAMP can use a different port (8889)
+  # MAMP users should also turn on Allow network access to MySQL otherwise MySQL might not be accessible
+  port: 3306
+  charset: utf8mb4
+  # The timezone configured on the MySQL server. This can be 'local', 'Z', or an offset in the form +HH:MM or -HH:MM
+  timezone: local
+
+verp:
+  # Enable to start an MX server that detects bounced messages using VERP
+  # In most cases you do not want to use it
+  # Requires root privileges
+  enabled: false
+  port: 2525
+  host: 0.0.0.0
+  # With DMARC, the Return-Path and From address must match the same domain.
+  # By default we get around this by using the VERP address in the Sender header,
+  # with the side effect that some email clients diplay an ugly on behalf of message.
+  # You can safely disable this Sender header if you're not using DMARC or your
+  # VERP hostname is in the same domain as the From address.
+  # disablesenderheader: true
+
+ldap:
+  # enable to use ldap user backend
+  enabled: false
+  # method is ldapjs or ldapauth - it chooses the library to be used. If not given, it chooses the one present.
+  # method: ldapjs
+  host: localhost
+  port: 3002
+  baseDN: ou=users,dc=company
+  filter: (|(username={{username}})(mail={{username}}))
+  # Username field in LDAP (uid/cn/username)
+  uidTag: username
+  # nameTag identifies the attribute to be used for user's full name
+  nameTag: username
+  passwordresetlink:
+  newUserRole: master
+  # Global namespace id
+  newUserNamespaceId: 1
+  # Use a different user to bind LDAP (final bind DN will be: {{uidTag}}: {{bindUser}},{{baseDN}})
+  bindUser: name@company.net
+  bindPassword: mySecretPassword
+
+postfixbounce:
+  # Enable to allow writing Postfix bounce log to Mailtrain listener
+  # If enabled, tail mail.log to Mailtrain with the following command:
+  #     tail -f -n +0 /var/log/mail.log | nc localhost 5699 -
+  enabled: false
+  port: 5699
+  # allow connections from localhost only
+  host: 127.0.0.1
+
+# extra options for nodemailer
+nodemailer:
+  #textEncoding: base64
+
+queue:
+  # How many parallel sender processes to spawn
+  processes: 2
+
+cors:
+  # Allow subscription widgets to be embedded
+  # origins: ['https://www.example.com']
+
+mosaico:
+  # Installed templates
+  fsTemplates:
+  - key: versafix-1
+    label: Versafix One
+  # Inject custom scripts
+  # customscripts:
+  # - /mosaico/custom/my-mosaico-plugin.js
+
+grapesjs:
+# Installed templates
+  templates:
+    - key: demo
+      label: HTML Template
+    - key: aves
+      label: MJML Template
+
+reports:
+  # The whole reporting functionality can be disabled below if the they are not needed and the DB cannot be
+  # properly protected.
+  # Reports rely on custom user defined Javascript snippets defined in the report template. The snippets are run on the
+  # server when generating a report. As these snippets are stored in the DB, they pose a security risk because they can
+  # help gaining access to the server if the DB cannot
+  # be properly protected (e.g. if it is shared with another application with security weaknesses).
+  # Mailtrain mitigates this problem by running the custom Javascript snippets in a chrooted environment and under a
+  # DB user that cannot modify the database (see userRO in [mysql] above). However the chrooted environment is available
+  # only if Mailtrain is started as root. The chrooted environment still does not prevent the custom JS script in
+  # performing network operations and in generating XSS attacks as part of the report.
+  # The bottom line is that if people who are creating report templates or have write access to the DB cannot be trusted,
+  # then it's safer to switch off the reporting functionality below.
+  enabled: false
+
+testServer:
+  # Starts a vanity server that redirects all mail to /dev/null
+  # Mostly needed for local development
+  enabled: false
+  port: 5587
+  mailboxServerPort: 3001
+  host: 0.0.0.0
+  username: testuser
+  password: testpass
+  logger: false
+
+seleniumWebDriver:
+  browser: phantomjs
+
+
+roles:
+  global:
+    master:
+      name: Master
+      admin: true
+      description: All permissions
+      permissions: [rebuildPermissions, createJavascriptWithROAccess, manageBlacklist, manageSettings, setupAutomation]
+      rootNamespaceRole: master
+
+  namespace:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, createNamespace, createList, createCustomForm, createReport, createReportTemplate, createTemplate, createMosaicoTemplate, createSendConfiguration, createCampaign, manageUsers]
+      children:
+        sendConfiguration: [viewPublic, viewPrivate, edit, delete, share, sendWithoutOverrides, sendWithAllowedOverrides, sendWithAnyOverrides]
+        list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports]
+        customForm: [view, edit, delete, share]
+        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats]
+        template: [view, edit, delete, share, viewFiles, manageFiles]
+        report: [view, edit, delete, share, execute, viewContent, viewOutput]
+        reportTemplate: [view, edit, delete, share, execute]
+        mosaicoTemplate: [view, edit, delete, share, viewFiles, manageFiles]
+        namespace: [view, edit, delete, share, createNamespace, createList, createCustomForm, createReport, createReportTemplate, createTemplate, createMosaicoTemplate, createSendConfiguration, createCampaign, manageUsers]
+
+  sendConfiguration:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [viewPublic, viewPrivate, edit, delete, share, sendWithoutOverrides, sendWithAllowedOverrides, sendWithAnyOverrides]
+
+  list:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports]
+
+  customForm:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share]
+
+  campaign:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, manageMessages]
+
+  template:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, viewFiles, manageFiles]
+
+  report:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, execute, viewContent, viewOutput]
+
+  reportTemplate:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, execute]
+
+  mosaicoTemplate:
+    master:
+      name: Master
+      description: All permissions
+      permissions: [view, edit, delete, share, viewFiles, manageFiles]
+
+

server/config/docker-production.toml.tmpl

@@ -0,0 +1,9 @@
+[mysql]
+host="mysql"
+
+[redis]
+enabled=true
+host="redis"
+
+[reports]
+enabled=true

server/config/test.toml

@@ -0,0 +1,24 @@
+[www]
+port=3000
+[mysql]
+user="mailtrain_test"
+password="bahquaiphoor"
+database="mailtrain_test"
+[testServer]
+enabled=true
+[seleniumWebDriver]
+browser="phantomjs"
+[ldap]
+# enable to use ldap user backend
+enabled=false
+host="localhost"
+port=3002
+baseDN="ou=users,dc=example"
+filter="(|(username={{username}})(mail={{username}}))"
+#Username field in LDAP (uid/cn/username)
+uidTag="username"
+# nameTag identifies the attribute to be used for user's full name
+nameTag="username"
+passwordresetlink="xxx"
+[reports]
+enabled=true

server/index.js

@@ -0,0 +1,126 @@
+'use strict';
+
+const config = require('config');
+const log = require('./lib/log');
+const appBuilder = require('./app-builder');
+const http = require('http');
+const triggers = require('./services/triggers');
+const importer = require('./lib/importer');
+const feedcheck = require('./lib/feedcheck');
+const verpServer = require('./services/verp-server');
+const testServer = require('./services/test-server');
+const postfixBounceServer = require('./services/postfix-bounce-server');
+const tzupdate = require('./services/tzupdate');
+const dbcheck = require('./lib/dbcheck');
+const senders = require('./lib/senders');
+const reportProcessor = require('./lib/report-processor');
+const executor = require('./lib/executor');
+const privilegeHelpers = require('./lib/privilege-helpers');
+const knex = require('./lib/knex');
+const shares = require('./models/shares');
+const { AppType } = require('../shared/app');
+
+const trustedPort = config.www.trustedPort;
+const sandboxPort = config.www.sandboxPort;
+const publicPort = config.www.publicPort;
+const host = config.www.host;
+
+if (config.title) {
+    process.title = config.title;
+}
+
+
+function startHTTPServer(appType, appName, port, callback) {
+    const app = appBuilder.createApp(appType);
+    app.set('port', port);
+
+    const server = http.createServer(app);
+
+    server.on('error', err => {
+        if (err.syscall !== 'listen') {
+            throw err;
+        }
+
+        const bind = typeof port === 'string' ? 'Pipe ' + port : 'Port ' + port;
+
+        // handle specific listen errors with friendly messages
+        switch (err.code) {
+            case 'EACCES':
+                log.error('Express', '%s requires elevated privileges', bind);
+                return process.exit(1);
+            case 'EADDRINUSE':
+                log.error('Express', '%s is already in use', bind);
+                return process.exit(1);
+            default:
+                throw err;
+        }
+    });
+
+    server.on('listening', () => {
+        const addr = server.address();
+        const bind = typeof addr === 'string' ? 'pipe ' + addr : 'port ' + addr.port;
+        log.info('Express', 'WWW server [%s] listening on %s', appName, bind);
+    });
+
+    server.listen({port, host}, callback);
+}
+
+
+// ---------------------------------------------------------------------------------------
+// Start the whole circus here
+// ---------------------------------------------------------------------------------------
+dbcheck(err => { // Check if database needs upgrading before starting the server - legacy migration first
+    if (err) {
+        log.error('DB', err.message || err);
+        return process.exit(1);
+    }
+
+    knex.migrate.latest() // And now the current migration with Knex
+
+    .then(() => shares.regenerateRoleNamesTable())
+    .then(() => shares.rebuildPermissions())
+
+/*
+    .then(() =>
+        testServer(() => {
+            senders.spawn(() => {
+            });
+        })
+    );
+*/
+
+    .then(() =>
+        executor.spawn(() => {
+            testServer(() => {
+                verpServer(() => {
+                    startHTTPServer(AppType.TRUSTED, 'trusted', trustedPort, () => {
+                        startHTTPServer(AppType.SANDBOXED, 'sandbox', sandboxPort, () => {
+                            startHTTPServer(AppType.PUBLIC, 'public', publicPort, () => {
+                                privilegeHelpers.dropRootPrivileges();
+
+                                tzupdate.start();
+
+                                importer.spawn(() => {
+                                    feedcheck.spawn(() => {
+                                        senders.spawn(() => {
+                                            triggers.start();
+
+                                            postfixBounceServer(async () => {
+                                                (async () => {
+                                                    await reportProcessor.init();
+                                                    log.info('Service', 'All services started');
+                                                })();
+                                            });
+                                        });
+                                    });
+                                });
+                            });
+                        });
+                    });
+                });
+            });
+        })
+    );
+});
+
+

server/lib/campaign-sender.js

@@ -0,0 +1,407 @@
+'use strict';
+
+const config = require('config');
+const mailers = require('./mailers');
+const knex = require('./knex');
+const subscriptions = require('../models/subscriptions');
+const contextHelpers = require('./context-helpers');
+const campaigns = require('../models/campaigns');
+const templates = require('../models/templates');
+const lists = require('../models/lists');
+const fields = require('../models/fields');
+const sendConfigurations = require('../models/send-configurations');
+const links = require('../models/links');
+const {CampaignSource, CampaignType} = require('../../shared/campaigns');
+const {SubscriptionStatus} = require('../../shared/lists');
+const tools = require('./tools');
+const request = require('request-promise');
+const files = require('../models/files');
+const htmlToText = require('html-to-text');
+const {getPublicUrl} = require('./urls');
+const blacklist = require('../models/blacklist');
+const libmime = require('libmime');
+
+
+class CampaignSender {
+    constructor() {
+    }
+
+    static async testSend(context, listCid, subscriptionCid, campaignId, sendConfigurationId, html, text) {
+        let sendConfiguration, list, fieldsGrouped, campaign, subscriptionGrouped, useVerp, useVerpSenderHeader, mergeTags, attachments;
+
+        await knex.transaction(async tx => {
+            sendConfiguration = await sendConfigurations.getByIdTx(tx, context, sendConfigurationId, false, true);
+            list = await lists.getByCidTx(tx, context, listCid);
+            fieldsGrouped = await fields.listGroupedTx(tx, list.id);
+
+            useVerp = config.verp.enabled && sendConfiguration.verp_hostname;
+            useVerpSenderHeader = this.useVerp && config.verp.disablesenderheader !== true;
+
+            subscriptionGrouped = await subscriptions.getByCid(context, list.id, subscriptionCid);
+            mergeTags = fields.getMergeTags(fieldsGrouped, subscriptionGrouped);
+
+            if (campaignId) {
+                campaign = await campaigns.getByIdTx(tx, context, campaignId, false, campaigns.Content.WITHOUT_SOURCE_CUSTOM);
+            } else {
+                // This is to fake the campaign for getMessageLinks, which is called inside formatMessage
+                campaign = {
+                    cid: '[CAMPAIGN_ID]'
+                };
+            }
+        });
+
+        const encryptionKeys = [];
+        for (const fld of fieldsGrouped) {
+            if (fld.type === 'gpg' && mergeTags[fld.key]) {
+                encryptionKeys.push(mergeTags[fld.key].trim());
+            }
+        }
+
+        attachments = [];
+        // replace data: images with embedded attachments
+        html = html.replace(/(<img\b[^>]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => {
+            const cid = shortid.generate() + '-attachments';
+            attachments.push({
+                path: dataUri,
+                cid
+            });
+            return prefix + 'cid:' + cid;
+        });
+
+        html = tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, html, true);
+
+        text = (text || '').trim()
+            ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, text)
+            : htmlToText.fromString(html, {wordwrap: 130});
+
+
+        const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
+
+        const getOverridable = key => {
+            return sendConfiguration[key];
+        }
+
+        const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
+
+        const mail = {
+            from: {
+                name: getOverridable('from_name'),
+                address: getOverridable('from_email')
+            },
+            replyTo: getOverridable('reply_to'),
+            xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
+            to: {
+                name: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false),
+                address: subscriptionGrouped.email
+            },
+            sender: this.useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false,
+
+            envelope: this.useVerp ? {
+                from: campaignAddress + '@' + sendConfiguration.verp_hostname,
+                to: subscriptionGrouped.email
+            } : false,
+
+            headers: {
+                'x-fbl': campaignAddress,
+                // custom header for SparkPost
+                'x-msys-api': JSON.stringify({
+                    campaign_id: campaignAddress
+                }),
+                // custom header for SendGrid
+                'x-smtpapi': JSON.stringify({
+                    unique_args: {
+                        campaign_id: campaignAddress
+                    }
+                }),
+                // custom header for Mailgun
+                'x-mailgun-variables': JSON.stringify({
+                    campaign_id: campaignAddress
+                }),
+                'List-ID': {
+                    prepared: true,
+                    value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>'
+                }
+            },
+            list: {
+                unsubscribe: null
+            },
+            subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false),
+            html,
+            text,
+
+            attachments,
+            encryptionKeys
+        };
+
+
+        let response;
+        try {
+            const info = await mailer.sendMassMail(mail);
+            response = info.response || info.messageId;
+        } catch (err) {
+            response = err.response || err.message;
+        }
+
+        return response;
+    }
+
+
+    async init(settings) {
+        this.listsById = new Map(); // listId -> list
+        this.listsByCid = new Map(); // listCid -> list
+        this.listsFieldsGrouped = new Map(); // listId -> fieldsGrouped
+        this.attachments = [];
+
+        await knex.transaction(async tx => {
+            if (settings.campaignCid) {
+                this.campaign = await campaigns.rawGetByTx(tx, 'cid', settings.campaignCid);
+            } else {
+                this.campaign = await campaigns.rawGetByTx(tx, 'id', settings.campaignId);
+            }
+
+            const campaign = this.campaign;
+
+            this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration, false, true);
+
+            for (const listSpec of campaign.lists) {
+                const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list);
+                this.listsById.set(list.id, list);
+                this.listsByCid.set(list.cid, list);
+                this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id));
+            }
+
+            if (campaign.source === CampaignSource.TEMPLATE) {
+                this.template = await templates.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.data.sourceTemplate, false);
+            }
+
+            const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', campaign.id);
+            for (const attachment of attachments) {
+                this.attachments.push({
+                    filename: attachment.originalname,
+                    path: files.getFilePath('campaign', 'attachment', campaign.id, attachment.filename)
+                });
+            }
+
+        });
+
+        this.useVerp = config.verp.enabled && sendConfiguration.verp_hostname;
+        this.useVerpSenderHeader = this.useVerp && config.verp.disablesenderheader !== true;
+    }
+
+    async _getMessage(campaign, list, subscriptionGrouped, mergeTags, replaceDataImgs) {
+        let html = '';
+        let text = '';
+        let renderTags = false;
+
+        if (campaign.source === CampaignSource.URL) {
+            const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
+            for (const key in mergeTags) {
+                form[key] = mergeTags[key];
+            }
+
+            const response = await request.post({
+                uri: campaign.sourceUrl,
+                form,
+                resolveWithFullResponse: true
+            });
+
+            if (response.statusCode !== 200) {
+                throw new Error(`Received status code ${httpResponse.statusCode} from ${campaign.sourceUrl}`);
+            }
+
+            html = response.body;
+            text = '';
+            renderTags = false;
+
+        } else if (campaign.source === CampaignSource.CUSTOM || campaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN || campaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
+            html = campaign.data.sourceCustom.html;
+            text = campaign.data.sourceCustom.text;
+            renderTags = true;
+
+        } else if (campaign.source === CampaignSource.TEMPLATE) {
+            const template = this.template;
+            html = template.html;
+            text = template.text;
+            renderTags = true;
+        }
+
+        html = await links.updateLinks(campaign, list, subscriptionGrouped, mergeTags, html);
+
+        const attachments = this.attachments.slice();
+        if (replaceDataImgs) {
+            // replace data: images with embedded attachments
+            html = html.replace(/(<img\b[^>]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => {
+                const cid = shortid.generate() + '-attachments';
+                attachments.push({
+                    path: dataUri,
+                    cid
+                });
+                return prefix + 'cid:' + cid;
+            });
+        }
+
+        html = renderTags ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, html, true) : html;
+
+        text = (text || '').trim()
+            ? (renderTags ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, text) : text)
+            : htmlToText.fromString(html, {wordwrap: 130});
+
+        return {
+            html,
+            text,
+            attachments
+        };
+    }
+
+    _getExtraTags(campaign) {
+        const tags = {};
+
+        if (campaign.type === CampaignType.RSS_ENTRY) {
+            const rssEntry = campaign.data.rssEntry;
+            tags['RSS_ENTRY_TITLE'] = rssEntry.title;
+            tags['RSS_ENTRY_DATE'] = rssEntry.date;
+            tags['RSS_ENTRY_LINK'] = rssEntry.link;
+            tags['RSS_ENTRY_CONTENT'] = rssEntry.content;
+            tags['RSS_ENTRY_SUMMARY'] = rssEntry.summary;
+            tags['RSS_ENTRY_IMAGE_URL'] = rssEntry.image_url;
+        }
+
+        return tags;
+    }
+
+    async getMessage(listCid, subscriptionCid) {
+        const list = this.listsByCid.get(listCid);
+        const subscriptionGrouped = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid);
+        const flds = this.listsFieldsGrouped.get(list.id);
+        const campaign = this.campaign;
+        const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign));
+
+        return await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, false);
+    }
+
+    async sendMessage(listId, email) {
+        if (await blacklist.isBlacklisted(email)) {
+            return;
+        }
+
+        const list = this.listsById.get(listId);
+        const subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, email);
+        const flds = this.listsFieldsGrouped.get(listId);
+        const campaign = this.campaign;
+
+        const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign));
+
+        const encryptionKeys = [];
+        for (const fld of flds) {
+            if (fld.type === 'gpg' && mergeTags[fld.key]) {
+                encryptionKeys.push(mergeTags[fld.key].trim());
+            }
+        }
+
+        const sendConfiguration = this.sendConfiguration;
+
+        const {html, text, attachments} = await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, true);
+
+        const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
+
+        let listUnsubscribe = null;
+        if (!list.listunsubscribe_disabled) {
+            listUnsubscribe = campaign.unsubscribe_url
+                ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, campaign.unsubscribe_url)
+                : getPublicUrl('/subscription/' + list.cid + '/unsubscribe/' + subscriptionGrouped.cid);
+        }
+
+        const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
+
+        await mailer.throttleWait();
+
+        const getOverridable = key => {
+            if (sendConfiguration[key + '_overridable'] && this.campaign[key + '_override'] !== null) {
+                return campaign[key + '_override'];
+            } else {
+                return sendConfiguration[key];
+            }
+        }
+
+        const mail = {
+            from: {
+                name: getOverridable('from_name'),
+                address: getOverridable('from_email')
+            },
+            replyTo: getOverridable('reply_to'),
+            xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
+            to: {
+                name: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false),
+                address: subscriptionGrouped.email
+            },
+            sender: this.useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false,
+
+            envelope: this.useVerp ? {
+                from: campaignAddress + '@' + sendConfiguration.verp_hostname,
+                to: subscriptionGrouped.email
+            } : false,
+
+            headers: {
+                'x-fbl': campaignAddress,
+                // custom header for SparkPost
+                'x-msys-api': JSON.stringify({
+                    campaign_id: campaignAddress
+                }),
+                // custom header for SendGrid
+                'x-smtpapi': JSON.stringify({
+                    unique_args: {
+                        campaign_id: campaignAddress
+                    }
+                }),
+                // custom header for Mailgun
+                'x-mailgun-variables': JSON.stringify({
+                    campaign_id: campaignAddress
+                }),
+                'List-ID': {
+                    prepared: true,
+                    value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>'
+                }
+            },
+            list: {
+                unsubscribe: listUnsubscribe
+            },
+            subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false),
+            html,
+            text,
+
+            attachments,
+            encryptionKeys
+        };
+
+
+        let status;
+        let response;
+        try {
+            const info = await mailer.sendMassMail(mail);
+            status = SubscriptionStatus.SUBSCRIBED;
+            response = info.response || info.messageId;
+
+            await knex('campaigns').where('id', campaign.id).increment('delivered');
+        } catch (err) {
+            status = SubscriptionStatus.BOUNCED;
+            response = err.response || err.message;
+            await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced');
+        }
+
+        const responseId = response.split(/\s+/).pop();
+
+        const now = new Date();
+        await knex('campaign_messages').insert({
+            campaign: this.campaign.id,
+            list: listId,
+            subscription: subscriptionGrouped.id,
+            send_configuration: sendConfiguration.id,
+            status,
+            response,
+            response_id: responseId,
+            updated: now
+        });
+    }
+}
+
+module.exports = CampaignSender;

server/lib/client-helpers.js

@@ -0,0 +1,50 @@
+'use strict';
+
+const passport = require('./passport');
+const config = require('config');
+const forms = require('../models/forms');
+const shares = require('../models/shares');
+const urls = require('./urls');
+
+
+async function getAnonymousConfig(context, appType) {
+    return {
+        authMethod: passport.authMethod,
+        isAuthMethodLocal: passport.isAuthMethodLocal,
+        externalPasswordResetLink: config.ldap.passwordresetlink,
+        language: config.language || 'en',
+        isAuthenticated: !!context.user,
+        trustedUrlBase: urls.getTrustedUrlBase(),
+        trustedUrlBaseDir: urls.getTrustedUrlBaseDir(),
+        sandboxUrlBase: urls.getSandboxUrlBase(),
+        sandboxUrlBaseDir: urls.getSandboxUrlBaseDir(),
+        publicUrlBase: urls.getPublicUrlBase(),
+        publicUrlBaseDir: urls.getPublicUrlBaseDir(),
+        appType
+    }
+}
+
+async function getAuthenticatedConfig(context) {
+    const globalPermissions = {};
+    for (const perm of shares.getGlobalPermissions(context)) {
+        globalPermissions[perm] = true;
+    }
+
+    return {
+        defaultCustomFormValues: await forms.getDefaultCustomFormValues(),
+        user: {
+            id: context.user.id,
+            username: context.user.username,
+            namespace: context.user.namespace
+        },
+        globalPermissions,
+        editors: config.editors,
+        mosaico: config.mosaico,
+        verpEnabled: config.verp.enabled
+    }
+}
+
+
+module.exports.getAuthenticatedConfig = getAuthenticatedConfig;
+module.exports.getAnonymousConfig = getAnonymousConfig;
+

server/lib/context-helpers.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const knex = require('./knex');
+
+function getRequestContext(req) {
+    const context = {
+        user: req.user
+    };
+
+    return context;
+}
+
+const adminContext = {
+    user: {
+        admin: true,
+        id: 0,
+        username: '',
+        name: '',
+        email: ''
+    }
+};
+
+function getAdminContext() {
+    return adminContext;
+}
+
+module.exports = {
+    getRequestContext,
+    getAdminContext
+};

server/lib/dbcheck.js

@@ -0,0 +1,232 @@
+'use strict';
+
+/*
+This module handles Mailtrain database initialization and upgrades
+*/
+
+const config = require('config');
+const mysql = require('mysql2');
+const log = require('./log');
+const fs = require('fs');
+const pathlib = require('path');
+const Handlebars = require('handlebars');
+
+const highestLegacySchemaVersion = 33;
+
+const mysqlConfig = {
+    multipleStatements: true
+};
+Object.keys(config.mysql).forEach(key => mysqlConfig[key] = config.mysql[key]);
+const db = mysql.createPool(mysqlConfig);
+
+function listTables(callback) {
+    db.getConnection((err, connection) => {
+        if (err) {
+            if (err.code === 'ER_ACCESS_DENIED_ERROR') {
+                err = new Error('Could not access the database. Check MySQL config and authentication credentials');
+            }
+            if (err.code === 'ECONNREFUSED' || err.code === 'PROTOCOL_SEQUENCE_TIMEOUT') {
+                err = new Error('Could not connect to the database. Check MySQL host and port configuration');
+            }
+            return callback(err);
+        }
+        let query = 'SHOW TABLES';
+        connection.query(query, (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+            let tables = {};
+            [].concat(rows || []).forEach(row => {
+                let name;
+                let table;
+                Object.keys(row).forEach(key => {
+                    if (/^Tables_in_/i.test(key)) {
+                        table = name = row[key];
+                    }
+                });
+                if (/__\d+$/.test(name)) {
+                    let parts = name.split('__');
+                    parts.pop();
+                    table = parts.join('__');
+                }
+                if (tables.hasOwnProperty(table)) {
+                    tables[table].push(name);
+                } else {
+                    tables[table] = [name];
+                }
+                return table;
+            });
+            return callback(null, tables);
+        });
+    });
+}
+
+function getSchemaVersion(callback) {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SHOW TABLES LIKE "knex_migrations"', (err, rows) => {
+            if (err) {
+                return callback(err);
+            }
+
+            if (rows.length > 0) {
+                connection.release();
+
+                callback(null, highestLegacySchemaVersion);
+            } else {
+                connection.query('SELECT `value` FROM `settings` WHERE `key`=?', ['db_schema_version'], (err, rows) => {
+                    connection.release();
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    let dbSchemaVersion = rows && rows[0] && Number(rows[0].value) || 0;
+                    callback(null, dbSchemaVersion);
+                });
+            }
+        });
+    });
+}
+
+function listUpdates(current, callback) {
+    current = current || 0;
+    fs.readdir(pathlib.join(__dirname, '..', 'setup', 'sql'), (err, list) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let updates = [];
+        [].concat(list || []).forEach(row => {
+            if (/^upgrade-\d+\.sql$/i.test(row)) {
+                let seq = row.match(/\d+/)[0];
+                if (seq > current) {
+                    updates.push({
+                        seq: Number(seq),
+                        path: pathlib.join(__dirname, '..', 'setup', 'sql', row)
+                    });
+                }
+            }
+        });
+
+        return callback(null, updates);
+    });
+}
+
+function getSql(path, data, callback) {
+    fs.readFile(path, 'utf-8', (err, source) => {
+        if (err) {
+            return callback(err);
+        }
+        const rendered = data ? Handlebars.compile(source)(data) : source;
+        return callback(null, rendered);
+    });
+}
+
+function runInitial(callback) {
+    let dump = process.env.NODE_ENV === 'test' ? 'mailtrain-test.sql' : 'mailtrain.sql';
+    let fname = process.env.DB_FROM_START ? 'base.sql' : dump;
+    let path = pathlib.join(__dirname, '..', 'setup', 'sql', fname);
+    log.info('sql', 'Loading tables from %s', fname);
+    applyUpdate({
+        path
+    }, callback);
+}
+
+function runUpdates(callback, runCount) {
+    runCount = Number(runCount) || 0;
+    listTables((err, tables) => {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!tables.settings) {
+            if (runCount) {
+                return callback(new Error('Settings table not found from database'));
+            }
+            log.info('sql', 'SQL not set up, initializing');
+            return runInitial(runUpdates.bind(null, callback, ++runCount));
+        }
+
+        getSchemaVersion((err, schemaVersion) => {
+            if (err) {
+                return callback(err);
+            }
+
+            if (schemaVersion >= highestLegacySchemaVersion) {
+                // nothing to do here, already updated
+                return callback(null, false);
+            }
+
+            listUpdates(schemaVersion, (err, updates) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                let pos = 0;
+                let runNext = () => {
+                    if (pos >= updates.length) {
+                        return callback(null, pos);
+                    }
+                    let update = updates[pos++];
+                    update.data = {
+                        tables
+                    };
+                    applyUpdate(update, (err, status) => {
+                        if (err) {
+                            return callback(err);
+                        }
+                        if (status) {
+                            log.info('sql', 'Update %s applied', update.seq);
+                        } else {
+                            log.info('sql', 'Update %s not applied', update.seq);
+                        }
+
+                        runNext();
+                    });
+                };
+
+                runNext();
+            });
+        });
+
+    });
+}
+
+function applyUpdate(update, callback) {
+    getSql(update.path, update.data, (err, sql) => {
+        if (err) {
+            return callback(err);
+        }
+
+        db.getConnection((err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            connection.query(sql, err => {
+                connection.release();
+                if (err) {
+                    return callback(err);
+                }
+
+                return callback(null, true);
+            });
+        });
+    });
+}
+
+module.exports = callback => {
+    runUpdates(err => {
+        if (err) {
+            return callback(err);
+        }
+        db.end(() => {
+            log.info('sql', 'Database check completed');
+            return callback(null, true);
+        });
+    });
+};

server/lib/dependency-helpers.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const knex = require('./knex');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const entitySettings = require('./entity-settings');
+const shares = require('../models/shares');
+const { enforce } = require('./helpers');
+
+const defaultNoOfDependenciesReported = 20;
+
+async function ensureNoDependencies(tx, context, id, depSpecs) {
+
+    const deps = [];
+    let andMore = false;
+
+    for (const depSpec of depSpecs) {
+        const entityType = entitySettings.getEntityType(depSpec.entityTypeId);
+
+        let rows;
+
+        if (depSpec.query) {
+            rows = await depSpec.query(tx).limit(defaultNoOfDependenciesReported + 1);
+        } else if (depSpec.column) {
+            rows = await tx(entityType.entitiesTable).where(depSpec.column, id).select(['id', 'name']).limit(defaultNoOfDependenciesReported + 1);
+        } else if (depSpec.rows) {
+            rows = await depSpec.rows(tx, defaultNoOfDependenciesReported + 1)
+        }
+
+        for (const row of rows) {
+            if (deps.length === defaultNoOfDependenciesReported) {
+                andMore = true;
+                break;
+            }
+
+            if (await shares.checkEntityPermissionTx(tx, context, depSpec.entityTypeId, row.id, 'view')) {
+                deps.push({
+                    entityTypeId: depSpec.entityTypeId,
+                    name: row.name,
+                    link: entityType.clientLink(row.id)
+                });
+            } else {
+                deps.push({
+                    entityTypeId: depSpec.entityTypeId,
+                    id: row.id
+                });
+            }
+        }
+    }
+
+    if (deps.length > 0) {
+        throw new interoperableErrors.DependencyPresentError('', {
+            dependencies: deps,
+            andMore
+        });
+    }
+}
+
+module.exports.ensureNoDependencies = ensureNoDependencies;

server/lib/dt-helpers.js

@@ -0,0 +1,189 @@
+'use strict';
+
+const knex = require('./knex');
+const entitySettings = require('./entity-settings');
+
+async function ajaxListTx(tx, params, queryFun, columns, options) {
+    options = options || {};
+
+    const columnsNames = [];
+    const columnsSelect = [];
+
+    for (const col of columns) {
+        if (typeof col === 'string') {
+            columnsNames.push(col);
+            columnsSelect.push(col);
+        } else {
+            columnsNames.push(col.name);
+
+            if (col.raw) {
+                columnsSelect.push(tx.raw(col.raw));
+            } else if (col.query) {
+                columnsSelect.push(function () { return col.query(this); });
+            }
+        }
+    }
+
+    if (params.operation === 'getBy') {
+        const query = queryFun(tx);
+        query.whereIn(columnsNames[parseInt(params.column)], params.values);
+        query.select(columnsSelect);
+        query.options({rowsAsArray:true});
+
+        const rows = await query;
+        const rowsOfArray = rows.map(row => Object.keys(row).map(key => row[key]));
+        return rowsOfArray;
+
+    } else {
+        const whereFun = function() {
+            let searchVal = '%' + params.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
+            for (let colIdx = 0; colIdx < params.columns.length; colIdx++) {
+                const col = params.columns[colIdx];
+                if (col.searchable) {
+                    this.orWhere(columnsNames[parseInt(col.data)], 'like', searchVal);
+                }
+            }
+        }
+
+        /* There are a few SQL peculiarities that make this query a bit weird:
+           - Group by (which is used in getting permissions) don't go well with count(*). Thus we run the actual query
+             as a sub-query and then count the number of results.
+           - SQL does not like if it have columns with the same name in the subquery. This happens multiple tables are joined.
+             To circumvent this, we select only the first column (whatever it is). Since this is not "distinct", it is supposed
+             to give us the right number of rows anyway.
+         */
+        const recordsTotalQuery = tx.count('* as recordsTotal').from(function () { return queryFun(this).select(columnsSelect[0]).as('records'); }).first();
+        const recordsTotal = (await recordsTotalQuery).recordsTotal;
+
+        const recordsFilteredQuery = tx.count('* as recordsFiltered').from(function () { return queryFun(this).select(columnsSelect[0]).where(whereFun).as('records'); }).first();
+        const recordsFiltered = (await recordsFilteredQuery).recordsFiltered;
+
+        const query = queryFun(tx);
+        query.where(whereFun);
+
+        query.offset(parseInt(params.start));
+
+        const limit = parseInt(params.length);
+        if (limit >= 0) {
+            query.limit(limit);
+        }
+
+        query.select([...columnsSelect, ...options.extraColumns || [] ]);
+
+        for (const order of params.order) {
+            if (options.orderByBuilder) {
+                options.orderByBuilder(query, columnsNames[params.columns[order.column].data], order.dir);
+            } else {
+                query.orderBy(columnsNames[params.columns[order.column].data], order.dir);
+            }
+        }
+
+        query.options({rowsAsArray:true});
+
+        const rows = await query;
+        const rowsOfArray = rows.map(row => {
+            const arr = Object.keys(row).map(field => row[field]);
+
+            if (options.mapFun) {
+                const result = options.mapFun(arr);
+                return result || arr;
+            } else {
+                return arr;
+            }
+        });
+
+        const result = {
+            draw: params.draw,
+            recordsTotal,
+            recordsFiltered,
+            data: rowsOfArray
+        };
+
+        return result;
+    }
+}
+
+async function ajaxListWithPermissionsTx(tx, context, fetchSpecs, params, queryFun, columns, options) {
+    // Note that this function is not intended to be used with the synthetic admin context obtained by contextHelpers.getAdminContext()
+    options = options || {};
+
+    const permCols = [];
+    for (const fetchSpec of fetchSpecs) {
+        const entityType = entitySettings.getEntityType(fetchSpec.entityTypeId);
+        const entityIdColumn = fetchSpec.column ? fetchSpec.column : entityType.entitiesTable + '.id';
+
+        permCols.push({
+            name: `permissions_${fetchSpec.entityTypeId}`,
+            query: builder => builder
+                .from(entityType.permissionsTable)
+                .select(knex.raw('GROUP_CONCAT(operation SEPARATOR \';\')'))
+                .whereRaw(`${entityType.permissionsTable}.entity = ${entityIdColumn}`)
+                .where(`${entityType.permissionsTable}.user`, context.user.id)
+                .as(`permissions_${fetchSpec.entityTypeId}`)
+        });
+    }
+
+    return await ajaxListTx(
+        tx,
+        params,
+        builder => {
+            let query = queryFun(builder);
+
+            for (const fetchSpec of fetchSpecs) {
+                const entityType = entitySettings.getEntityType(fetchSpec.entityTypeId);
+
+                if (fetchSpec.requiredOperations) {
+                    const entityIdColumn = fetchSpec.column ? fetchSpec.column : entityType.entitiesTable + '.id';
+
+                    query = query.innerJoin(
+                        function () {
+                            return this.from(entityType.permissionsTable).distinct('entity').where('user', context.user.id).whereIn('operation', fetchSpec.requiredOperations).as(`permitted__${fetchSpec.entityTypeId}`);
+                        },
+                        `permitted__${fetchSpec.entityTypeId}.entity`, entityIdColumn)
+                }
+            }
+
+            return query;
+        },
+        [
+            ...columns,
+            ...permCols
+        ],
+        {
+            mapFun: data => {
+                for (let idx = 0; idx < fetchSpecs.length; idx++) {
+                    data[columns.length + idx] = data[columns.length + idx].split(';');
+                }
+
+                if (options.mapFun) {
+                    const result = options.mapFun(data);
+                    return result || data;
+                } else {
+                    return data;
+                }
+            },
+
+            orderByBuilder: options.orderByBuilder,
+            extraColumns: options.extraColumns
+        }
+    );
+}
+
+async function ajaxList(params, queryFun, columns, options) {
+    return await knex.transaction(async tx => {
+        return await ajaxListTx(tx, params, queryFun, columns, options)
+    });
+}
+
+async function ajaxListWithPermissions(context, fetchSpecs, params, queryFun, columns, options) {
+    return await knex.transaction(async tx => {
+        return await ajaxListWithPermissionsTx(tx, context, fetchSpecs, params, queryFun, columns, options)
+    });
+}
+
+module.exports = {
+    ajaxListTx,
+    ajaxList,
+    ajaxListWithPermissionsTx,
+    ajaxListWithPermissions
+};

server/lib/entity-settings.js

@@ -0,0 +1,151 @@
+'use strict';
+
+const ReplacementBehavior = {
+    NONE: 1,
+    REPLACE: 2,
+    RENAME: 3
+};
+
+const entityTypes = {
+    namespace: {
+        entitiesTable: 'namespaces',
+        sharesTable: 'shares_namespace',
+        permissionsTable: 'permissions_namespace',
+        clientLink: id => `/namespaces/${id}`
+    },
+    list: {
+        entitiesTable: 'lists',
+        sharesTable: 'shares_list',
+        permissionsTable: 'permissions_list',
+        clientLink: id => `/lists/${id}`
+    },
+    customForm: {
+        entitiesTable: 'custom_forms',
+        sharesTable: 'shares_custom_form',
+        permissionsTable: 'permissions_custom_form',
+        clientLink: id => `/lists/forms/${id}`
+    },
+    campaign: {
+        entitiesTable: 'campaigns',
+        sharesTable: 'shares_campaign',
+        permissionsTable: 'permissions_campaign',
+        dependentPermissions: {
+            extraColumns: ['parent'],
+            getParent: entity => entity.parent
+        },
+        files: {
+            file: {
+                table: 'files_campaign_file',
+                permissions: {
+                    view: 'viewFiles',
+                    manage: 'manageFiles'
+                },
+                defaultReplacementBehavior: ReplacementBehavior.REPLACE
+            },
+            attachment: {
+                table: 'files_campaign_attachment',
+                permissions: {
+                    view: 'viewAttachments',
+                    manage: 'manageAttachments'
+                },
+                defaultReplacementBehavior: ReplacementBehavior.NONE
+            }
+        },
+        clientLink: id => `/campaigns/${id}`
+    },
+    template: {
+        entitiesTable: 'templates',
+        sharesTable: 'shares_template',
+        permissionsTable: 'permissions_template',
+        files: {
+            file: {
+                table: 'files_template_file',
+                permissions: {
+                    view: 'viewFiles',
+                    manage: 'manageFiles'
+                },
+                defaultReplacementBehavior: ReplacementBehavior.REPLACE
+            }
+        },
+        clientLink: id => `/templates/${id}`
+    },
+    sendConfiguration: {
+        entitiesTable: 'send_configurations',
+        sharesTable: 'shares_send_configuration',
+        permissionsTable: 'permissions_send_configuration',
+        clientLink: id => `/send-configurations/${id}`
+    },
+    report: {
+        entitiesTable: 'reports',
+        sharesTable: 'shares_report',
+        permissionsTable: 'permissions_report',
+        clientLink: id => `/reports/${id}`
+    },
+    reportTemplate: {
+        entitiesTable: 'report_templates',
+        sharesTable: 'shares_report_template',
+        permissionsTable: 'permissions_report_template',
+        clientLink: id => `/reports/templates/${id}`
+    },
+    mosaicoTemplate: {
+        entitiesTable: 'mosaico_templates',
+        sharesTable: 'shares_mosaico_template',
+        permissionsTable: 'permissions_mosaico_template',
+        files: {
+            file: {
+                table: 'files_mosaico_template_file',
+                permissions: {
+                    view: 'viewFiles',
+                    manage: 'manageFiles'
+                },
+                defaultReplacementBehavior: ReplacementBehavior.REPLACE
+            },
+            block: {
+                table: 'files_mosaico_template_block',
+                permissions: {
+                    view: 'viewFiles',
+                    manage: 'manageFiles'
+                },
+                defaultReplacementBehavior: ReplacementBehavior.REPLACE
+            }
+        },
+        clientLink: id => `/templates/mosaico/${id}`
+    },
+    user: {
+        entitiesTable: 'users',
+        clientLink: id => `/users/${id}`
+    }
+};
+
+const entityTypesWithPermissions = {};
+for (const key in entityTypes) {
+    if (entityTypes[key].permissionsTable) {
+        entityTypesWithPermissions[key] = entityTypes[key];
+    }
+}
+
+
+function getEntityTypes() {
+    return entityTypes;
+}
+
+function getEntityTypesWithPermissions() {
+    return entityTypesWithPermissions;
+}
+
+function getEntityType(entityTypeId) {
+    const entityType = entityTypes[entityTypeId];
+
+    if (!entityType) {
+        throw new Error(`Unknown entity type ${entityTypeId}`);
+    }
+
+    return entityType
+}
+
+module.exports = {
+    getEntityTypes,
+    getEntityTypesWithPermissions,
+    getEntityType,
+    ReplacementBehavior
+}

server/lib/executor.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const fork = require('child_process').fork;
+const log = require('./log');
+const path = require('path');
+
+const requestCallbacks = {};
+let messageTid = 0;
+let executorProcess;
+
+module.exports = {
+    spawn,
+    start,
+    stop
+};
+
+function spawn(callback) {
+    log.verbose('Executor', 'Spawning executor process');
+
+    executorProcess = fork(path.join(__dirname, '..', 'services', 'executor.js'), [], {
+        cwd: path.join(__dirname, '..'),
+        env: {NODE_ENV: process.env.NODE_ENV}
+    });
+
+    executorProcess.on('message', msg => {
+        if (msg) {
+            if (msg.type === 'process-started') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.startedCallback) {
+                    requestCallback.startedCallback(msg.tid, );
+                }
+
+            } else if (msg.type === 'process-failed') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.failedCallback) {
+                    requestCallback.failedCallback(msg.msg);
+                }
+
+                delete requestCallbacks[msg.tid];
+
+            } else if (msg.type === 'process-finished') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.startedCallback) {
+                    requestCallback.finishedCallback(msg.code, msg.signal);
+                }
+
+                delete requestCallbacks[msg.tid];
+
+            } else if (msg.type === 'executor-started') {
+                log.info('Executor', 'Executor process started.');
+                return callback();
+            }
+        }
+    });
+
+    executorProcess.on('close', (code, signal) => {
+        log.info('Executor', 'Executor process exited with code %s signal %s', code, signal);
+    });
+}
+
+function start(type, data, startedCallback, finishedCallback, failedCallback) {
+    requestCallbacks[messageTid] = {
+        startedCallback,
+        finishedCallback,
+        failedCallback
+    };
+
+    executorProcess.send({
+        type: 'start-' + type,
+        data,
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+function stop(tid) {
+    executorProcess.send({
+        type: 'stop-process',
+        tid
+    });
+}
+

server/lib/feedcheck.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const fork = require('child_process').fork;
+const log = require('./log');
+const path = require('path');
+const senders = require('./senders');
+
+let feedcheckProcess;
+
+module.exports = {
+    spawn
+};
+
+function spawn(callback) {
+    log.verbose('Feed', 'Spawning feedcheck process');
+
+    feedcheckProcess = fork(path.join(__dirname, '..', 'services', 'feedcheck.js'), [], {
+        cwd: path.join(__dirname, '..'),
+        env: {NODE_ENV: process.env.NODE_ENV}
+    });
+
+    feedcheckProcess.on('message', msg => {
+        if (msg) {
+            if (msg.type === 'feedcheck-started') {
+                log.info('Feed', 'Feedcheck process started');
+                return callback();
+            } else if (msg.type === 'entries-added') {
+                senders.scheduleCheck();
+            }
+        }
+    });
+
+    feedcheckProcess.on('close', (code, signal) => {
+        log.error('Feed', 'Feedcheck process exited with code %s signal %s', code, signal);
+    });
+}

server/lib/file-helpers.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const passport = require('./passport');
+const files = require('../models/files');
+
+const path = require('path');
+const uploadedFilesDir = path.join(files.filesDir, 'uploaded');
+const {castToInteger} = require('./helpers');
+
+const multer = require('multer')({
+    dest: uploadedFilesDir
+});
+
+function installUploadHandler(router, url, replacementBehavior, type, subType, transformResponseFn) {
+    router.postAsync(url, passport.loggedIn, multer.array('files[]'), async (req, res) => {
+        return res.json(await files.createFiles(req.context, type || req.params.type, subType || req.params.subType, castToInteger(req.params.entityId), req.files, replacementBehavior, transformResponseFn));
+    });
+}
+
+module.exports = {
+    installUploadHandler
+};

server/lib/helpers.js

@@ -0,0 +1,39 @@
+'use strict';
+
+module.exports = {
+    enforce,
+    cleanupFromPost,
+    filterObject,
+    castToInteger
+};
+
+function enforce(condition, message) {
+    if (!condition) {
+        throw new Error(message);
+    }
+}
+
+function cleanupFromPost(value) {
+    return (value || '').toString().trim();
+}
+
+function filterObject(obj, allowedKeys) {
+    const result = {};
+    for (const key in obj) {
+        if (allowedKeys.has(key)) {
+            result[key] = obj[key];
+        }
+    }
+
+    return result;
+}
+
+function castToInteger(id) {
+    const val = parseInt(id);
+
+    if (!Number.isInteger(val)) {
+        throw new Error('Invalid id');
+    }
+
+    return val;
+}

server/lib/importer.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const knex = require('./knex');
+const fork = require('child_process').fork;
+const log = require('./log');
+const path = require('path');
+const {ImportStatus, RunStatus} = require('../../shared/imports');
+
+let messageTid = 0;
+let importerProcess;
+
+module.exports = {
+    spawn,
+    scheduleCheck
+};
+
+function spawn(callback) {
+    log.verbose('Importer', 'Spawning importer process');
+
+    knex.transaction(async tx => {
+        await tx('imports').where('status', ImportStatus.PREP_RUNNING).update({status: ImportStatus.PREP_SCHEDULED});
+        await tx('imports').where('status', ImportStatus.PREP_STOPPING).update({status: ImportStatus.PREP_FAILED});
+
+        await tx('imports').where('status', ImportStatus.RUN_RUNNING).update({status: ImportStatus.RUN_SCHEDULED});
+        await tx('imports').where('status', ImportStatus.RUN_STOPPING).update({status: ImportStatus.RUN_FAILED});
+
+        await tx('import_runs').where('status', RunStatus.RUNNING).update({status: RunStatus.SCHEDULED});
+        await tx('import_runs').where('status', RunStatus.STOPPING).update({status: RunStatus.FAILED});
+
+    }).then(() => {
+        importerProcess = fork(path.join(__dirname, '..', 'services', 'importer.js'), [], {
+            cwd: path.join(__dirname, '..'),
+            env: {NODE_ENV: process.env.NODE_ENV}
+        });
+
+        importerProcess.on('message', msg => {
+            if (msg) {
+                if (msg.type === 'importer-started') {
+                    log.info('Importer', 'Importer process started');
+                    return callback();
+                }
+            }
+        });
+
+        importerProcess.on('close', (code, signal) => {
+            log.error('Importer', 'Importer process exited with code %s signal %s', code, signal);
+        });
+    });
+}
+
+function scheduleCheck() {
+    importerProcess.send({
+        type: 'scheduleCheck',
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+

server/lib/knex.js

@@ -0,0 +1,14 @@
+'use strict';
+
+const config = require('config');
+
+const knex = require('server/lib/knex')({
+    client: 'mysql2',
+    connection: config.mysql,
+    migrations: {
+        directory: __dirname + '/../setup/knex/migrations'
+    }
+    //, debug: true
+});
+
+module.exports = knex;

server/lib/log.js

@@ -0,0 +1,8 @@
+'use strict';
+
+const config = require('config');
+const log = require('npmlog');
+
+log.level = config.log.level;
+
+module.exports = log;

server/lib/mailers.js

@@ -0,0 +1,232 @@
+'use strict';
+
+const log = require('./log');
+const config = require('config');
+
+const Handlebars = require('handlebars');
+const util = require('util');
+const nodemailer = require('nodemailer');
+const aws = require('aws-sdk');
+const openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
+const sendConfigurations = require('../models/send-configurations');
+
+const contextHelpers = require('./context-helpers');
+const settings = require('../models/settings');
+const tools = require('./tools');
+const htmlToText = require('html-to-text');
+
+const bluebird = require('bluebird');
+
+const _ = require('./translate')._;
+
+const transports = new Map();
+
+async function getOrCreateMailer(sendConfigurationId) {
+    let sendConfiguration;
+
+    if (!sendConfiguration) {
+        sendConfiguration = await sendConfigurations.getSystemSendConfiguration();
+    } else {
+        sendConfiguration = await sendConfigurations.getById(contextHelpers.getAdminContext(), sendConfigurationId, false, true);
+    }
+
+    const transport = transports.get(sendConfiguration.id) || await _createTransport(sendConfiguration);
+    return transport.mailer;
+}
+
+function invalidateMailer(sendConfigurationId) {
+    transports.delete(sendConfigurationId);
+}
+
+
+
+
+async function _sendMail(transport, mail, template) {
+    let tryCount = 0;
+    const trySend = (callback) => {
+        tryCount++;
+        transport.sendMail(mail, (err, info) => {
+            if (err) {
+                log.error('Mail', err);
+                if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) {
+                    // temporary error, try again
+                    log.verbose('Mail', 'Retrying after %s sec. ...', tryCount);
+                    return setTimeout(trySend, tryCount * 1000);
+                }
+                return callback(err);
+            }
+            return callback(null, info);
+        });
+    };
+
+    const trySendAsync = bluebird.promisify(trySend);
+    return await trySendAsync();
+}
+
+async function _sendTransactionalMail(transport, mail, template) {
+    if (!mail.headers) {
+        mail.headers = {};
+    }
+    mail.headers['X-Sending-Zone'] = 'transactional';
+
+    const htmlRenderer = await tools.getTemplate(template.html);
+
+    if (htmlRenderer) {
+        mail.html = htmlRenderer(template.data || {});
+    }
+
+    const preparedHtml = await tools.prepareHtml(mail.html);
+
+    if (preparedHtml) {
+        mail.html = preparedHtml;
+    }
+
+    const textRenderer = await tools.getTemplate(template.text);
+
+    if (textRenderer) {
+        mail.text = textRenderer(template.data || {});
+    } else if (mail.html) {
+        mail.text = htmlToText.fromString(mail.html, {
+            wordwrap: 130
+        });
+    }
+
+    return await _sendMail(transport, mail);
+}
+
+async function _createTransport(sendConfiguration) {
+    const mailerSettings = sendConfiguration.mailer_settings;
+    const mailerType = sendConfiguration.mailer_type;
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey', 'pgpPassphrase']);
+
+    const existingTransport = transports.get(sendConfiguration.id);
+
+    let existingListeners = [];
+    if (existingTransport) {
+        existingListeners = existingTransport.listeners('idle');
+        existingTransport.removeAllListeners('idle');
+        existingTransport.removeAllListeners('stream');
+        existingTransport.throttleWait = null;
+    }
+
+    const logFunc = (...args) => {
+        const level = args.shift();
+        args.shift();
+        args.unshift('Mail');
+        log[level](...args);
+    };
+
+
+    let transportOptions;
+
+    if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
+        transportOptions = {
+            pool: true,
+            host: mailerSettings.hostname,
+            port: mailerSettings.port || false,
+            secure: mailerSettings.encryption === 'TLS',
+            ignoreTLS: mailerSettings.encryption === 'NONE',
+            auth: mailerSettings.useAuth ? {
+                user: mailerSettings.user,
+                pass: mailerSettings.password
+            } : false,
+            debug: mailerSettings.logTransactions,
+            logger: mailerSettings.logTransactions ? {
+                debug: logFunc.bind(null, 'verbose'),
+                info: logFunc.bind(null, 'info'),
+                error: logFunc.bind(null, 'error')
+            } : false,
+            maxConnections: mailerSettings.maxConnections,
+            maxMessages: mailerSettings.maxMessages,
+            tls: {
+                rejectUnauthorized: !mailerSettings.allowSelfSigned
+            }
+        };
+
+    } else if (mailerType === sendConfigurations.MailerType.AWS_SES) {
+        const sendingRate = mailerSettings.throttling / 3600;  // convert to messages/second
+
+        transportOptions = {
+            SES: new aws.SES({
+                apiVersion: '2010-12-01',
+                accessKeyId: mailerSettings.key,
+                secretAccessKey: mailerSettings.secret,
+                region: mailerSettings.region
+            }),
+            debug: mailerSettings.logTransactions,
+            logger: mailerSettings.logTransactions ? {
+                debug: logFunc.bind(null, 'verbose'),
+                info: logFunc.bind(null, 'info'),
+                error: logFunc.bind(null, 'error')
+            } : false,
+            maxConnections: mailerSettings.maxConnections,
+            sendingRate
+        };
+
+    } else {
+        throw new Error('Invalid mail transport');
+    }
+
+    const transport = nodemailer.createTransport(transportOptions, config.nodemailer);
+
+    transport.use('stream', openpgpEncrypt({
+        signingKey: configItems.pgpPrivateKey,
+        passphrase: configItems.pgpPassphrase
+    }));
+
+    if (existingListeners.length) {
+        log.info('Mail', 'Reattaching %s idle listeners', existingListeners.length);
+        existingListeners.forEach(listener => transport.on('idle', listener));
+    }
+
+    let throttleWait;
+
+    if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
+        let throttling = mailerSettings.throttling;
+        if (throttling) {
+            throttling = 1 / (throttling / (3600 * 1000));
+        }
+
+        let lastCheck = Date.now();
+
+        throttleWait = function (next) {
+            if (!throttling) {
+                return next();
+            }
+            let nextCheck = Date.now();
+            let checkDiff = (nextCheck - lastCheck);
+            if (checkDiff < throttling) {
+                log.verbose('Mail', 'Throttling next message in %s sec.', (throttling - checkDiff) / 1000);
+                setTimeout(() => {
+                    lastCheck = Date.now();
+                    next();
+                }, throttling - checkDiff);
+            } else {
+                lastCheck = nextCheck;
+                next();
+            }
+        };
+    } else {
+        throttleWait = next => next();
+    }
+
+    transport.mailer = {
+        throttleWait: bluebird.promisify(throttleWait),
+        sendTransactionalMail: async (mail, template) => await _sendTransactionalMail(transport, mail, template),
+        sendMassMail: async (mail, template) => await _sendMail(transport, mail)
+    };
+
+    transports.set(sendConfiguration.id, transport);
+    return transport;
+}
+
+class MailerError extends Error {
+    constructor(msg, responseCode) {
+        super(msg);
+        this.responseCode = responseCode;
+    }
+}
+
+module.exports.getOrCreateMailer = getOrCreateMailer;
+module.exports.invalidateMailer = invalidateMailer;
+module.exports.MailerError = MailerError;

server/lib/namespace-helpers.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const { enforce } = require('./helpers');
+const shares = require('../models/shares');
+const interoperableErrors = require('../../shared/interoperable-errors');
+
+async function validateEntity(tx, entity) {
+    enforce(entity.namespace, 'Entity namespace not set');
+    if (!await tx('namespaces').where('id', entity.namespace).first()) {
+        throw new interoperableErrors.NamespaceNotFoundError();
+    }
+}
+
+async function validateMove(context, entity, existing, entityTypeId, createOperation, deleteOperation) {
+    if (existing.namespace !== entity.namespace) {
+        await shares.enforceEntityPermission(context, 'namespace', entity.namespace, createOperation);
+        await shares.enforceEntityPermission(context, entityTypeId, entity.id, deleteOperation);
+    }
+}
+
+module.exports = {
+    validateEntity,
+    validateMove
+};

server/lib/nodeify.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const nodeify = require('server/lib/nodeify');
+
+module.exports.nodeifyPromise = nodeify;
+
+module.exports.nodeifyFunction = (asyncFun) => {
+    return (...args) => {
+        const callback = args.pop();
+
+        const promise = asyncFun(...args);
+
+        return module.exports.nodeifyPromise(promise, callback);
+    };
+};

server/lib/passport.js

@@ -0,0 +1,229 @@
+'use strict';
+
+const config = require('config');
+const log = require('./log');
+const _ = require('./translate')._;
+const util = require('util');
+
+const passport = require('server/lib/passport');
+const LocalStrategy = require('passport-local').Strategy;
+
+const csrf = require('csurf');
+const bodyParser = require('body-parser');
+
+const users = require('../models/users');
+const { nodeifyFunction, nodeifyPromise } = require('./nodeify');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const contextHelpers = require('./context-helpers');
+
+let authMode = 'local';
+
+let LdapStrategy;
+let ldapStrategyOpts;
+if (config.ldap.enabled) {
+    if (!config.ldap.method || config.ldap.method == 'ldapjs') {
+        try {
+            LdapStrategy = require('passport-ldapjs').Strategy; // eslint-disable-line global-require
+            authMode = 'ldapjs';
+            log.info('LDAP', 'Found module "passport-ldapjs". It will be used for LDAP auth.');
+
+            ldapStrategyOpts = {
+                server: {
+                    url: 'ldap://' + config.ldap.host + ':' + config.ldap.port
+                },
+                base: config.ldap.baseDN,
+                search: {
+                    filter: config.ldap.filter,
+                    attributes: [config.ldap.uidTag, config.ldap.nameTag, 'mail'],
+                    scope: 'sub'
+                },
+                uidTag: config.ldap.uidTag,
+                bindUser: config.ldap.bindUser,
+                bindPassword: config.ldap.bindPassword
+            };
+
+        } catch (exc) {
+            log.info('LDAP', 'Module "passport-ldapjs" not installed. It will not be used for LDAP auth.');
+        }
+    }
+
+    if (!LdapStrategy && (!config.ldap.method || config.ldap.method == 'ldapauth')) {
+        try {
+            LdapStrategy = require('passport-ldapauth').Strategy; // eslint-disable-line global-require
+            authMode = 'ldapauth';
+            log.info('LDAP', 'Found module "passport-ldapauth". It will be used for LDAP auth.');
+
+            ldapStrategyOpts = {
+                server: {
+                    url: 'ldap://' + config.ldap.host + ':' + config.ldap.port,
+                    searchBase: config.ldap.baseDN,
+                    searchFilter: config.ldap.filter,
+                    searchAttributes: [config.ldap.uidTag, config.ldap.nameTag, 'mail'],
+                    bindDN: config.ldap.bindUser,
+                    bindCredentials: config.ldap.bindPassword
+                },
+            };
+        } catch (exc) {
+            log.info('LDAP', 'Module "passport-ldapauth" not installed. It will not be used for LDAP auth.');
+        }
+    }
+}
+
+module.exports.csrfProtection = csrf({
+    cookie: true
+});
+
+module.exports.parseForm = bodyParser.urlencoded({
+    extended: false,
+    limit: config.www.postSize
+});
+
+module.exports.loggedIn = (req, res, next) => {
+    if (!req.user) {
+        next(new interoperableErrors.NotLoggedInError());
+    } else {
+        next();
+    }
+};
+
+module.exports.authByAccessToken = (req, res, next) => {
+    if (!req.query.access_token) {
+        res.status(403);
+        res.json({
+            error: 'Missing access_token',
+            data: []
+        });
+    }
+
+    users.getByAccessToken(req.query.access_token).then(user => {
+        req.user = user;
+        next();
+    }).catch(err => {
+        if (err instanceof interoperableErrors.PermissionDeniedError) {
+            res.status(403);
+            res.json({
+                error: 'Invalid or expired access_token',
+                data: []
+            });
+        } else {
+            res.status(500);
+            res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+    });
+};
+
+module.exports.tryAuthByRestrictedAccessToken = (req, res, next) => {
+    const pathComps = req.url.split('/');
+
+    pathComps.shift();
+    const restrictedAccessToken = pathComps.shift();
+    pathComps.unshift('');
+
+    const url = pathComps.join('/');
+
+    req.url = url;
+
+    users.getByRestrictedAccessToken(restrictedAccessToken).then(user => {
+        req.user = user;
+        next();
+    }).catch(err => {
+        next();
+    });
+};
+
+
+module.exports.setupRegularAuth = app => {
+    app.use(passport.initialize());
+    app.use(passport.session());
+};
+
+module.exports.restLogout = (req, res) => {
+    req.logout();
+    res.json();
+};
+
+module.exports.restLogin = (req, res, next) => {
+    passport.authenticate(authMode, (err, user, info) => {
+        if (err) {
+            return next(err);
+        }
+
+        if (!user) {
+            return next(new interoperableErrors.IncorrectPasswordError());
+        }
+
+        req.logIn(user, err => {
+            if (err) {
+                return next(err);
+            }
+
+            if (req.body.remember) {
+                // Cookie expires after 30 days
+                req.session.cookie.maxAge = 30 * 24 * 60 * 60 * 1000;
+            } else {
+                // Cookie expires at end of session
+                req.session.cookie.expires = false;
+            }
+
+            return res.json();
+        });
+    })(req, res, next);
+};
+
+if (LdapStrategy) {
+    log.info('Using LDAP auth (passport-' + authMode + ')');
+    module.exports.authMethod = 'ldap';
+    module.exports.isAuthMethodLocal = false;
+
+    passport.use(new LdapStrategy(ldapStrategyOpts, nodeifyFunction(async (profile) => {
+        try {
+            const user = await users.getByUsername(profile[config.ldap.uidTag]);
+
+            return {
+                id: user.id,
+                username: user.username,
+                name: profile[config.ldap.nameTag],
+                email: profile.mail,
+                role: user.role
+            };
+
+        } catch (err) {
+            if (err instanceof interoperableErrors.NotFoundError) {
+                const userId = await users.create(null, {
+                    username: profile[config.ldap.uidTag],
+                    role: config.ldap.newUserRole,
+                    namespace: config.ldap.newUserNamespaceId
+                });
+
+                return {
+                    id: userId,
+                    username: profile[config.ldap.uidTag],
+                    name: profile[config.ldap.nameTag],
+                    email: profile.mail,
+                    role: config.ldap.newUserRole
+                };
+            } else {
+                throw err;
+            }
+        }
+    })));
+
+    passport.serializeUser((user, done) => done(null, user));
+    passport.deserializeUser((user, done) => done(null, user));
+
+} else {
+    log.info('Using local auth');
+    module.exports.authMethod = 'local';
+    module.exports.isAuthMethodLocal = true;
+
+    passport.use(new LocalStrategy(nodeifyFunction(async (username, password) => {
+        return await users.getByUsernameIfPasswordMatch(username, password);
+    })));
+
+    passport.serializeUser((user, done) => done(null, user.id));
+    passport.deserializeUser((id, done) => nodeifyPromise(users.getById(contextHelpers.getAdminContext(), id), done));
+}
+

server/lib/privilege-helpers.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const log = require('./log');
+const config = require('config');
+
+const fs = require('fs');
+
+const tryRequire = require('try-require');
+const posix = tryRequire('posix');
+
+function _getConfigUidGid(userKey, groupKey, defaultUid, defaultGid) {
+    let uid = defaultUid;
+    let gid = defaultGid;
+
+    if (posix) {
+        try {
+            if (config[userKey]) {
+                uid = posix.getpwnam(config[userKey]).uid;
+            }
+        } catch (err) {
+            log.info('PrivilegeHelpers', 'Failed to resolve user id "%s"', config[userKey]);
+        }
+
+        try {
+            if (config[groupKey]) {
+                gid = posix.getpwnam(config[groupKey]).gid;
+            }
+        } catch (err) {
+            log.info('PrivilegeHelpers', 'Failed to resolve group id "%s"', config[groupKey]);
+        }
+    } else {
+        log.info('PrivilegeHelpers', 'Posix module not installed. Cannot resolve uid/gid');
+    }
+
+    return { uid, gid };
+}
+
+function getConfigUidGid() {
+    return _getConfigUidGid('user', 'group', process.getuid(), process.getgid());
+}
+
+function getConfigROUidGid() {
+    const rwIds = getConfigUidGid();
+    return _getConfigUidGid('roUser', 'roGroup', rwIds.uid, rwIds.gid);
+}
+
+function ensureMailtrainOwner(file, callback) {
+    const ids = getConfigUidGid();
+    fs.chown(file, ids.uid, ids.gid, callback);
+}
+
+function dropRootPrivileges() {
+    if (config.group) {
+        try {
+            process.setgid(config.group);
+            log.info('PrivilegeHelpers', 'Changed group to "%s" (%s)', config.group, process.getgid());
+        } catch (E) {
+            log.info('PrivilegeHelpers', 'Failed to change group to "%s" (%s)', config.group, E.message);
+        }
+    }
+
+    if (config.user) {
+        try {
+            process.setuid(config.user);
+            log.info('PrivilegeHelpers', 'Changed user to "%s" (%s)', config.user, process.getuid());
+        } catch (E) {
+            log.info('PrivilegeHelpers', 'Failed to change user to "%s" (%s)', config.user, E.message);
+        }
+    }
+}
+
+module.exports = {
+    dropRootPrivileges,
+    ensureMailtrainOwner,
+    getConfigUidGid,
+    getConfigROUidGid
+};

server/lib/report-helpers.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const path = require('path');
+
+function nameToFileName(name) {
+    return name.
+        trim().
+        toLowerCase().
+        replace(/[ .+/]/g, '-').
+        replace(/[^a-z0-9\-_]/gi, '').
+        replace(/--*/g, '-');
+}
+
+
+function getReportFileBase(report) {
+    return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + nameToFileName(report.name));
+}
+
+function getReportContentFile(report) {
+    return getReportFileBase(report) + '.out';
+}
+
+function getReportOutputFile(report) {
+    return getReportFileBase(report) + '.err';
+}
+
+
+module.exports = {
+    getReportContentFile,
+    getReportOutputFile,
+    nameToFileName
+};

server/lib/report-processor.js

@@ -0,0 +1,132 @@
+'use strict';
+
+const log = require('./log');
+const reports = require('../models/reports');
+const executor = require('./executor');
+const contextHelpers = require('./context-helpers');
+
+let runningWorkersCount = 0;
+let maxWorkersCount = 1;
+
+const workers = {};
+
+function startWorker(report) {
+
+    async function onStarted(tid) {
+        log.info('ReportProcessor', 'Worker process for "%s" started with tid %s. Current worker count is %s.', report.name, tid, runningWorkersCount);
+        workers[report.id] = tid;
+    }
+
+    async function onFinished(code, signal) {
+        runningWorkersCount--;
+        log.info('ReportProcessor', 'Worker process for "%s" (tid %s) exited with code %s signal %s. Current worker count is %s.', report.name, workers[report.id], code, signal, runningWorkersCount);
+        delete workers[report.id];
+
+        const fields = {};
+        if (code === 0) {
+            fields.state = reports.ReportState.FINISHED;
+            fields.last_run = new Date();
+        } else {
+            fields.state = reports.ReportState.FAILED;
+        }
+
+        try {
+            await reports.updateFields(report.id, fields);
+            setImmediate(tryStartWorkers);
+        } catch (err) {
+            log.error('ReportProcessor', err);
+        }
+    }
+
+    async function onFailed(msg) {
+        runningWorkersCount--;
+        log.error('ReportProcessor', 'Executing worker process for "%s" (tid %s) failed with message "%s". Current worker count is %s.', report.name, workers[report.id], msg, runningWorkersCount);
+        delete workers[report.id];
+
+        const fields = {
+            state: reports.ReportState.FAILED
+        };
+
+        try {
+            await reports.updateFields(report.id, fields);
+            setImmediate(tryStartWorkers);
+        } catch (err) {
+            log.error('ReportProcessor', err);
+        }
+    }
+
+    const reportData = {
+        id: report.id,
+        name: report.name
+    };
+
+    runningWorkersCount++;
+    executor.start('report-processor-worker', reportData, onStarted, onFinished, onFailed);
+}
+
+let isStartingWorkers = false;
+
+async function tryStartWorkers() {
+
+    if (isStartingWorkers) {
+        // Generally it is possible that this function is invoked simultaneously multiple times. This is to prevent it.
+        return;
+    }
+    isStartingWorkers = true;
+
+    try {
+        while (runningWorkersCount < maxWorkersCount) {
+            log.info('ReportProcessor', 'Trying to start worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+
+            const reportList = await reports.listByState(reports.ReportState.SCHEDULED, 1);
+
+            if (reportList.length > 0) {
+                log.info('ReportProcessor', 'Starting worker');
+
+                const report = reportList[0];
+                await reports.updateFields(report.id, {state: reports.ReportState.PROCESSING});
+                startWorker(report);
+
+            } else {
+                log.info('ReportProcessor', 'No more reports to start a worker for');
+                break;
+            }
+        }
+
+    } catch (err) {
+        log.error('ReportProcessor', err);
+    }
+
+    isStartingWorkers = false;
+}
+
+module.exports.start = async (reportId) => {
+    if (!workers[reportId]) {
+        log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
+        await reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, last_run: null});
+        await tryStartWorkers();
+    } else {
+        log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
+    }
+};
+
+module.exports.stop = async reportId => {
+    const tid = workers[reportId];
+    if (tid) {
+        log.info('ReportProcessor', 'Killing worker for report id: %s', reportId);
+        executor.stop(tid);
+
+        await reports.updateFields(reportId, { state: reports.ReportState.FAILED });
+    } else {
+        log.info('ReportProcessor', 'No running worker found for report id: %s', reportId);
+    }
+};
+
+module.exports.init = async () => {
+    try {
+        await reports.bulkChangeState(reports.ReportState.PROCESSING, reports.ReportState.SCHEDULED);
+        await tryStartWorkers();
+    } catch (err) {
+        log.error('ReportProcessor', err);
+    }
+};

server/lib/router-async.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const express = require('express');
+
+function replaceLastBySafeHandler(handlers) {
+    if (handlers.length === 0) {
+        return [];
+    }
+
+    const lastHandler = handlers[handlers.length - 1];
+    const ret = handlers.slice();
+    ret[handlers.length - 1] = (req, res, next) => lastHandler(req, res, next).catch(error => next(error));
+    return ret;
+}
+
+function create() {
+    const router = new express.Router();
+
+    router.allAsync = (path, ...handlers) => router.all(path, ...replaceLastBySafeHandler(handlers));
+    router.getAsync = (path, ...handlers) => router.get(path, ...replaceLastBySafeHandler(handlers));
+    router.postAsync = (path, ...handlers) => router.post(path, ...replaceLastBySafeHandler(handlers));
+    router.putAsync = (path, ...handlers) => router.put(path, ...replaceLastBySafeHandler(handlers));
+    router.deleteAsync = (path, ...handlers) => router.delete(path, ...replaceLastBySafeHandler(handlers));
+
+    return router;
+}
+
+module.exports = {
+    create
+};
+

server/lib/senders.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const fork = require('child_process').fork;
+const log = require('./log');
+const path = require('path');
+const knex = require('./knex');
+const {CampaignStatus} = require('../../shared/campaigns');
+
+let messageTid = 0;
+let senderProcess;
+
+function spawn(callback) {
+    log.verbose('Senders', 'Spawning master sender process');
+
+    knex('campaigns').where('status', CampaignStatus.SENDING).update({status: CampaignStatus.SCHEDULED})
+    .then(() => {
+        senderProcess = fork(path.join(__dirname, '..', 'services', 'sender-master.js'), [], {
+            cwd: path.join(__dirname, '..'),
+            env: {NODE_ENV: process.env.NODE_ENV}
+        });
+
+        senderProcess.on('message', msg => {
+            if (msg) {
+                if (msg.type === 'master-sender-started') {
+                    log.info('Senders', 'Master sender process started');
+                    return callback();
+                }
+            }
+        });
+
+        senderProcess.on('close', (code, signal) => {
+            log.error('Senders', 'Master sender process exited with code %s signal %s', code, signal);
+        });
+    });
+}
+
+function scheduleCheck() {
+    senderProcess.send({
+        type: 'schedule-check',
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+function reloadConfig(sendConfigurationId) {
+    senderProcess.send({
+        type: 'reload-config',
+        data: {
+            sendConfigurationId
+        },
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+module.exports = {
+    spawn,
+    scheduleCheck,
+    reloadConfig
+};
+

server/lib/subscription-mail-helpers.js

@@ -0,0 +1,164 @@
+'use strict';
+
+const log = require('npmlog');
+const fields = require('../models/fields');
+const settings = require('../models/settings');
+const {getTrustedUrl, getPublicUrl} = require('./urls');
+const { tUI, tMark } = require('./translate');
+const util = require('util');
+const contextHelpers = require('./context-helpers');
+const {getFieldColumn} = require('../../shared/lists');
+const forms = require('../models/forms');
+const mailers = require('./mailers');
+
+module.exports = {
+    sendAlreadySubscribed,
+    sendConfirmAddressChange,
+    sendConfirmSubscription,
+    sendConfirmUnsubscription,
+    sendSubscriptionConfirmed,
+    sendUnsubscriptionConfirmed
+};
+
+async function sendSubscriptionConfirmed(lang, list, email, subscription) {
+    const relativeUrls = {
+        preferencesUrl: '/subscription/' + list.cid + '/manage/' + subscription.cid,
+        unsubscribeUrl: '/subscription/' + list.cid + '/unsubscribe/' + subscription.cid
+    };
+
+    await _sendMail(list, email, 'subscription_confirmed', lang, tMark('subscription.confirmed'), relativeUrls, subscription);
+}
+
+async function sendAlreadySubscribed(lang, list, email, subscription) {
+    const relativeUrls = {
+        preferencesUrl: '/subscription/' + list.cid + '/manage/' + subscription.cid,
+        unsubscribeUrl: '/subscription/' + list.cid + '/unsubscribe/' + subscription.cid
+    };
+    await _sendMail(list, email, 'already_subscribed', lang, tMark('subscription.alreadyRegistered'), relativeUrls, subscription);
+}
+
+async function sendConfirmAddressChange(lang, list, email, cid, subscription) {
+    const relativeUrls = {
+        confirmUrl: '/subscription/confirm/change-address/' + cid
+    };
+    await _sendMail(list, email, 'confirm_address_change', lang, tMark('subscription.confirmEmailChange'), relativeUrls, subscription);
+}
+
+async function sendConfirmSubscription(lang, list, email, cid, subscription) {
+    const relativeUrls = {
+        confirmUrl: '/subscription/confirm/subscribe/' + cid
+    };
+    await _sendMail(list, email, 'confirm_subscription', lang, tMark('subscription.confirmSubscription'), relativeUrls, subscription);
+}
+
+async function sendConfirmUnsubscription(lang, list, email, cid, subscription) {
+    const relativeUrls = {
+        confirmUrl: '/subscription/confirm/unsubscribe/' + cid
+    };
+    await _sendMail(list, email, 'confirm_unsubscription', lang, tMark('subscription.confirmUnsubscription'), relativeUrls, subscription);
+}
+
+async function sendUnsubscriptionConfirmed(lang, list, email, subscription) {
+    const relativeUrls = {
+        subscribeUrl: '/subscription/' + list.cid + '?cid=' + subscription.cid
+    };
+    await _sendMail(list, email, 'unsubscription_confirmed', lang, tMark('subscription.unsubscriptionConfirmed'), relativeUrls, subscription);
+}
+
+function getDisplayName(flds, subscription) {
+    let firstName, lastName, name;
+
+    for (const fld of flds) {
+        if (fld.key === 'FIRST_NAME') {
+            firstName = subscription[fld.column];
+        }
+
+        if (fld.key === 'LAST_NAME') {
+            lastName = subscription[fld.column];
+        }
+
+        if (fld.key === 'NAME') {
+            name = subscription[fld.column];
+        }
+    }
+
+    if (name) {
+        return name;
+    } else if (firstName && lastName) {
+        return firstName + ' ' + lastName;
+    } else if (lastName) {
+        return lastName;
+    } else if (firstName) {
+        return firstName;
+    } else {
+        return '';
+    }
+}
+
+async function _sendMail(list, email, template, language, subjectKey, relativeUrls, subscription) {
+    const flds = await fields.list(contextHelpers.getAdminContext(), list.id);
+
+    const encryptionKeys = [];
+    for (const fld of flds) {
+        if (fld.type === 'gpg' && fld.value) {
+            encryptionKeys.push(subscription[getFieldColumn(fld)].value.trim());
+        }
+    }
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['defaultHomepage', 'adminEmail']);
+
+    const data = {
+        title: list.name,
+        homepage: configItems.defaultHomepage || getTrustedUrl(),
+        contactAddress: list.from_email || configItems.adminEmail,
+    };
+
+    for (let relativeUrlKey in relativeUrls) {
+        data[relativeUrlKey] = getPublicUrl(relativeUrls[relativeUrlKey], {language});
+    }
+
+    const fsTemplate = template.replace(/_/g, '-');
+    const text = {
+        template: 'subscription/mail-' + fsTemplate + '-text.hbs'
+    };
+
+    const html = {
+        template: 'subscription/mail-' + fsTemplate + '-html.mjml.hbs',
+        layout: 'subscription/layout.mjml.hbs',
+        type: 'mjml'
+    };
+
+    if (list.default_form) {
+        const form = await forms.getById(contextHelpers.getAdminContext(), list.default_form);
+
+        text.template = form['mail_' + template + '_text'] || text.template;
+        html.template = form['mail_' + template + '_html'] || html.template;
+        html.layout = form.layout || html.layout;
+    }
+
+    try {
+        if (list.send_configuration) {
+            const mailer = await mailers.getOrCreateMailer(list.send_configuration);
+            await mailer.sendTransactionalMail({
+                from: {
+                    name: configItems.defaultFrom,
+                    address: configItems.defaultAddress
+                },
+                to: {
+                    name: getDisplayName(flds, subscription),
+                    address: email
+                },
+                subject: tUI(subjectKey, language, { list: list.name }),
+                encryptionKeys
+            }, {
+                html,
+                text,
+                data
+            });
+        } else {
+            log.warn('Subscription', `Not sending email for list id:${list.id} because not send configuration is set.`);
+        }
+    } catch (err) {
+        log.error('Subscription', err);
+    }
+}

server/lib/tools.js

@@ -0,0 +1,193 @@
+'use strict';
+
+const util = require('util');
+const isemail = require('isemail');
+const path = require('path');
+const {getPublicUrl} = require('./urls');
+
+const bluebird = require('bluebird');
+
+const hasher = require('node-object-hash')();
+
+const mjml = require('mjml');
+const mjml2html = mjml.default;
+
+const hbs = require('hbs');
+const juice = require('juice');
+const he = require('he');
+
+const fs = require('fs-extra');
+
+const { JSDOM } = require('jsdom');
+const { tUI, tLog } = require('./translate');
+
+
+const templates = new Map();
+
+async function getTemplate(template) {
+    if (!template) {
+        return false;
+    }
+
+    const key = (typeof template === 'object') ? hasher.hash(template) : template;
+
+    if (templates.has(key)) {
+        return templates.get(key);
+    }
+
+    let source;
+    if (typeof template === 'object') {
+        source = await mergeTemplateIntoLayout(template.template, template.layout);
+    } else {
+        source = await fs.readFile(path.join(__dirname, '..', 'views', template), 'utf-8');
+    }
+
+    if (template.type === 'mjml') {
+        const compiled = mjml2html(source);
+
+        if (compiled.errors.length) {
+            throw new Error(compiled.errors[0].message || compiled.errors[0]);
+        }
+
+        source = compiled.html;
+    }
+
+    const renderer = hbs.handlebars.compile(source);
+    templates.set(key, renderer);
+
+    return renderer;
+}
+
+
+async function mergeTemplateIntoLayout(template, layout) {
+    layout = layout || '{{{body}}}';
+
+    async function readFile(relPath) {
+        return await fs.readFile(path.join(__dirname, '..', 'views', relPath), 'utf-8');
+    }
+
+    // Please dont end your custom messages with .hbs ...
+    if (layout.endsWith('.hbs')) {
+        layout = await readFile(layout);
+    }
+
+    if (template.endsWith('.hbs')) {
+        template = await readFile(template);
+    }
+
+    const source = layout.replace(/\{\{\{body\}\}\}/g, template);
+    return source;
+}
+
+async function validateEmail(address) {
+    const result = await new Promise(resolve => {
+        const result = isemail.validate(address, {
+            checkDNS: true,
+            errorLevel: 1
+        }, resolve);
+    });
+
+    return result;
+}
+
+function validateEmailGetMessage(result, address, language) {
+    let t;
+    if (language) {
+        t = (key, args) => tUI(key, language, args);
+    } else {
+        t = (key, args) => tLog(key, args);
+    }
+
+    if (result !== 0) {
+        switch (result) {
+            case 5:
+                return t('addressCheck.mxNotFound', {email: address});
+            case 6:
+                return t('addressCheck.domainNotFound', {email: address});
+            case 12:
+                return t('addressCheck.domainRequired', {email: address});
+            default:
+                return t('invalidEmailGeneric', {email: address});
+        }
+    }
+}
+
+function formatMessage(campaign, list, subscription, mergeTags, message, isHTML) {
+    const links = getMessageLinks(campaign, list, subscription);
+
+    const getValue = key => {
+        key = (key || '').toString().toUpperCase().trim();
+        if (links.hasOwnProperty(key)) {
+            return links[key];
+        }
+        if (mergeTags.hasOwnProperty(key)) {
+            const value = (mergeTags[key] || '').toString();
+            const containsHTML = /<[a-z][\s\S]*>/.test(value);
+            return isHTML ? he.encode((containsHTML ? value : value.replace(/(?:\r\n|\r|\n)/g, '<br/>')), {
+                useNamedReferences: true,
+                allowUnsafeSymbols: true
+            }) : (containsHTML ? htmlToText.fromString(value) : value);
+        }
+        return false;
+    };
+
+    return message.replace(/\[([a-z0-9_]+)(?:\/([^\]]+))?\]/ig, (match, identifier, fallback) => {
+        let value = getValue(identifier);
+        if (value === false) {
+            return match;
+        }
+        value = (value || fallback || '').trim();
+        return value;
+    });
+}
+
+async function prepareHtml(html) {
+    if (!(html || '').toString().trim()) {
+        return false;
+    }
+
+    const { window } = new JSDOM(html);
+
+    const head = window.document.querySelector('head');
+    let hasCharsetTag = false;
+    const metaTags = window.document.querySelectorAll('meta');
+    if (metaTags) {
+        for (let i = 0; i < metaTags.length; i++) {
+            if (metaTags[i].hasAttribute('charset')) {
+                metaTags[i].setAttribute('charset', 'utf-8');
+                hasCharsetTag = true;
+                break;
+            }
+        }
+    }
+    if (!hasCharsetTag) {
+        const charsetTag = window.document.createElement('meta');
+        charsetTag.setAttribute('charset', 'utf-8');
+        head.appendChild(charsetTag);
+    }
+    const preparedHtml = '<!doctype html><html>' + window.document.documentElement.innerHTML + '</html>';
+
+    return juice(preparedHtml);
+}
+
+function getMessageLinks(campaign, list, subscription) {
+    return {
+        LINK_UNSUBSCRIBE: getPublicUrl('/subscription/' + list.cid + '/unsubscribe/' + subscription.cid + '?c=' + campaign.cid),
+        LINK_PREFERENCES: getPublicUrl('/subscription/' + list.cid + '/manage/' + subscription.cid),
+        LINK_BROWSER: getPublicUrl('/archive/' + campaign.cid + '/' + list.cid + '/' + subscription.cid),
+        CAMPAIGN_ID: campaign.cid,
+        LIST_ID: list.cid,
+        SUBSCRIPTION_ID: subscription.cid
+    };
+}
+
+module.exports = {
+    validateEmail,
+    validateEmailGetMessage,
+    mergeTemplateIntoLayout,
+    getTemplate,
+    prepareHtml,
+    getMessageLinks,
+    formatMessage
+};
+

server/lib/translate.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const config = require('config');
+
+const i18n = require("i18next");
+const Backend = require("i18next-node-fs-backend");
+
+const path = require('path');
+
+i18n
+    .use(Backend)
+    // .use(Cache)
+    .init({
+        lng: config.language,
+
+        wait: true, // globally set to wait for loaded translations in translate hoc
+
+        // have a common namespace used around the full app
+        ns: ['common'],
+        defaultNS: 'common',
+
+        debug: true,
+
+        backend: {
+            loadPath: path.join(__dirname, 'locales/{{lng}}/{{ns}}.json')
+        }
+    })
+
+function tLog(key, args) {
+    if (!args) {
+        args = {};
+    }
+
+    return JSON.stringify([key, args]);
+}
+
+function tUI(key, lang, args) {
+    if (!args) {
+        args = {};
+    }
+
+    return i18n.t(key, { ...args, defaultValue, lng: lang });
+}
+
+function tMark(key) {
+    return key;
+}
+
+module.exports.tLog = tLog;
+module.exports.tUI = tUI;
+module.exports.tMark = tMark;

server/lib/urls.js

@@ -0,0 +1,71 @@
+'use strict';
+
+const config = require('config');
+const urllib = require('url');
+const {anonymousRestrictedAccessToken} = require('../../shared/urls');
+
+function getTrustedUrlBase() {
+    return urllib.resolve(config.www.trustedUrlBase, '');
+}
+
+function getSandboxUrlBase() {
+    return urllib.resolve(config.www.sandboxUrlBase, '');
+}
+
+function getPublicUrlBase() {
+    return urllib.resolve(config.www.publicUrlBase, '');
+}
+
+function _getUrl(urlBase, path, opts) {
+    const url = new URL(path || '', urlBase);
+
+    if (opts && opts.language) {
+        url.searchParams.append('lang', opts.language)
+    }
+
+    return url.toString();
+}
+
+function getTrustedUrl(path, opts) {
+    return _getUrl(config.www.trustedUrlBase, path || '', opts);
+}
+
+function getSandboxUrl(path, context, opts) {
+    if (context && context.user && context.user.restrictedAccessToken) {
+        return _getUrl(config.www.sandboxUrlBase, context.user.restrictedAccessToken + '/' + (path || ''), opts);
+    } else {
+        return _getUrl(config.www.sandboxUrlBase, anonymousRestrictedAccessToken + '/' + (path || ''), opts);
+    }
+}
+
+function getPublicUrl(path, opts) {
+    return _getUrl(config.www.publicUrlBase, path || '', opts);
+}
+
+
+function getTrustedUrlBaseDir() {
+    const mailtrainUrl = urllib.parse(config.www.trustedUrlBase);
+    return mailtrainUrl.pathname;
+}
+
+function getSandboxUrlBaseDir() {
+    const mailtrainUrl = urllib.parse(config.www.sandboxUrlBase);
+    return mailtrainUrl.pathname;
+}
+
+function getPublicUrlBaseDir() {
+    const mailtrainUrl = urllib.parse(config.www.publicUrlBase);
+    return mailtrainUrl.pathname;
+}
+
+module.exports = {
+    getTrustedUrl,
+    getSandboxUrl,
+    getPublicUrl,
+    getTrustedUrlBase,
+    getSandboxUrlBase,
+    getPublicUrlBase,
+    getTrustedUrlBaseDir,
+    getSandboxUrlBaseDir,
+    getPublicUrlBaseDir
+};

server/models/blacklist.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const dtHelpers = require('../lib/dt-helpers');
+const shares = require('./shares');
+const tools = require('../lib/tools');
+
+async function listDTAjax(context, params) {
+    shares.enforceGlobalPermission(context, 'manageBlacklist');
+
+    return await dtHelpers.ajaxList(
+        params,
+        builder => builder
+            .from('blacklist'),
+        ['blacklist.email']
+    );
+}
+
+async function search(context, offset, limit, search) {
+    return await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'manageBlacklist');
+
+        search = '%' + search + '%';
+
+        const count = await tx('blacklist').where('email', 'like', search).count('* as count').first().count;
+
+        const rows = await tx('blacklist').where('email', 'like', search).offset(offset).limit(limit);
+
+        return {
+            emails: rows.map(row => row.email),
+            total: count
+        };
+    });
+}
+
+async function add(context, email) {
+    return await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'manageBlacklist');
+
+        const existing = await tx('blacklist').where('email', email).first();
+        if (!existing) {
+            await tx('blacklist').insert({email});
+        }
+    });
+}
+
+async function remove(context, email) {
+    shares.enforceGlobalPermission(context, 'manageBlacklist');
+    await knex('blacklist').where('email', email).del();
+}
+
+async function isBlacklisted(email) {
+    const existing = await knex('blacklist').where('email', email).first();
+    return !!existing;
+}
+
+async function serverValidate(context, data) {
+    shares.enforceGlobalPermission(context, 'manageBlacklist');
+    const result = {};
+
+    if (data.email) {
+        const user = await knex('blacklist').where('email', data.email).first();
+
+        result.email = {};
+        result.email.invalid = await tools.validateEmail(data.email) !== 0;
+        result.email.exists = !!user;
+    }
+
+    return result;
+}
+
+module.exports.listDTAjax = listDTAjax;
+module.exports.add = add;
+module.exports.remove = remove;
+module.exports.search = search;
+module.exports.isBlacklisted = isBlacklisted;
+module.exports.serverValidate = serverValidate;

server/models/campaigns.js

@@ -0,0 +1,832 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const files = require('./files');
+const templates = require('./templates');
+const { CampaignStatus, CampaignSource, CampaignType, getSendConfigurationPermissionRequiredForSend} = require('../../shared/campaigns');
+const sendConfigurations = require('./send-configurations');
+const triggers = require('./triggers');
+const {SubscriptionStatus} = require('../../shared/lists');
+const subscriptions = require('./subscriptions');
+const segments = require('./segments');
+const senders = require('../lib/senders');
+
+const allowedKeysCommon = ['name', 'description', 'segment', 'namespace',
+    'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url'];
+
+const allowedKeysCreate = new Set(['type', 'source', ...allowedKeysCommon]);
+const allowedKeysCreateRssEntry = new Set(['type', 'source', 'parent', ...allowedKeysCommon]);
+const allowedKeysUpdate = new Set([...allowedKeysCommon]);
+
+const Content = {
+    ALL: 0,
+    WITHOUT_SOURCE_CUSTOM: 1,
+    ONLY_SOURCE_CUSTOM: 2,
+    RSS_ENTRY: 3,
+    SETTINGS_WITH_STATS: 4
+};
+
+function hash(entity, content) {
+    let filteredEntity;
+
+    if (content === Content.ALL) {
+        filteredEntity = filterObject(entity, allowedKeysUpdate);
+        filteredEntity.lists = entity.lists;
+
+    } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+        filteredEntity = filterObject(entity, allowedKeysUpdate);
+        filteredEntity.lists = entity.lists;
+        filteredEntity.data = {...filteredEntity.data};
+        delete filteredEntity.data.sourceCustom;
+
+    } else if (content === Content.ONLY_SOURCE_CUSTOM) {
+        filteredEntity = {
+            data: {
+                sourceCustom: entity.data.sourceCustom
+            }
+        };
+    }
+
+    return hasher.hash(filteredEntity);
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .whereNull('campaigns.parent'),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.status', 'campaigns.scheduled', 'campaigns.source', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+async function listChildrenDTAjax(context, campaignId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .where('campaigns.parent', campaignId),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.status', 'campaigns.scheduled', 'campaigns.source', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+
+async function listWithContentDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .whereIn('campaigns.source', [CampaignSource.CUSTOM, CampaignSource.CUSTOM_FROM_TEMPLATE, CampaignSource.CUSTOM_FROM_CAMPAIGN]),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+async function listOthersWhoseListsAreIncludedDTAjax(context, campaignId, listIds, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .whereNot('campaigns.id', campaignId)
+            .whereNotExists(qry => qry.from('campaign_lists').whereRaw('campaign_lists.campaign = campaigns.id').whereNotIn('campaign_lists.list', listIds)),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+async function listTestUsersDTAjax(context, campaignId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'view');
+
+        /*
+        This is supposed to produce queries like this:
+
+        select * from (
+          (select `subscription__1`.`email`, `subscription__1`.`cid`, 1 AS list, NULL AS segment from `subscription__1` where `subscription__1`.`status` = 1 and `subscription__1`.`is_test` = true)
+        UNION ALL
+          (select `subscription__2`.`email`, `subscription__2`.`cid`, 2 AS list, NULL AS segment from `subscription__2` where `subscription__2`.`status` = 1 and `subscription__2`.`is_test` = true)
+        ) as `test_subscriptions` inner join `lists` on `test_subscriptions`.`list` = `lists`.`id` inner join `segments` on `test_subscriptions`.`segment` = `segments`.`id`
+          inner join `namespaces` on `lists`.`namespace` = `namespaces`.`id`
+
+        This was too much for Knex, so we partially construct these queries directly as strings;
+        */
+
+        const subsQrys = [];
+        const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+
+        for (const cpgList of cpgLists) {
+            const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
+            const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
+
+            const sqlQry = knex.from(subsTable)
+                .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+                .where(subsTable + '.is_test', true)
+                .where(function() {
+                    addSegmentQuery(this);
+                })
+                .select([subsTable + '.email', subsTable + '.cid', knex.raw('? AS list', [cpgList.list]), knex.raw('? AS segment', [cpgList.segment])])
+                .toSQL().toNative();
+
+            subsQrys.push(sqlQry);
+        }
+
+        if (subsQrys.length > 0) {
+            let subsQry;
+
+            if (subsQrys.length === 1) {
+                const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`'
+                subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
+
+            } else {
+                const subsUnionSql = '(' +
+                    subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
+                    ') as `test_subscriptions`';
+                const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+                subsQry = knex.raw(subsUnionSql, subsUnionBindings);
+            }
+
+            return await dtHelpers.ajaxListWithPermissions(
+                context,
+                [{ entityTypeId: 'list', requiredOperations: ['viewSubscriptions'], column: 'subs.list_id' }],
+                params,
+                builder => {
+                    return builder.from(function () {
+                        return this.from(subsQry)
+                            .innerJoin('lists', 'test_subscriptions.list', 'lists.id')
+                            .innerJoin('namespaces', 'lists.namespace', 'namespaces.id')
+                            .select([
+                                knex.raw('CONCAT_WS(":", lists.cid, test_subscriptions.cid) AS cid'),
+                                'test_subscriptions.email', 'test_subscriptions.cid AS subscription_cid', 'lists.cid AS list_cid',
+                                'lists.name as list_name', 'namespaces.name AS namespace_name', 'lists.id AS list_id'
+                            ])
+                            .as('subs');
+                    });
+                },
+                [ 'subs.cid', 'subs.email', 'subs.subscription_cid', 'subs.list_cid', 'subs.list_name', 'subs.namespace_name' ]
+            );
+
+        } else {
+            const result = {
+                draw: params.draw,
+                recordsTotal: 0,
+                recordsFiltered: 0,
+                data: []
+            };
+
+            return result;
+        }
+    });
+}
+
+async function getTrackingSettingsByCidTx(tx, cid) {
+    const entity = await tx('campaigns').where('campaigns.cid', cid)
+        .select([
+            'campaigns.id', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled'
+        ])
+        .first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    return entity;
+}
+
+async function rawGetByTx(tx, key, id) {
+    const entity = await tx('campaigns').where('campaigns.' + key, id)
+        .leftJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
+        .groupBy('campaigns.id')
+        .select([
+            'campaigns.id', 'campaigns.cid', 'campaigns.name', 'campaigns.description', 'campaigns.namespace', 'campaigns.status', 'campaigns.type', 'campaigns.source',
+            'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject_override',
+            'campaigns.data', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled', 'campaigns.unsubscribe_url', 'campaigns.scheduled',
+            knex.raw(`GROUP_CONCAT(CONCAT_WS(\':\', campaign_lists.list, campaign_lists.segment) ORDER BY campaign_lists.id SEPARATOR \';\') as lists`)
+        ])
+        .first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    if (entity.lists) {
+        entity.lists = entity.lists.split(';').map(x => {
+            const entries = x.split(':');
+            const list = Number.parseInt(entries[0]);
+            const segment = entries[1] ? Number.parseInt(entries[1]) : null;
+            return {list, segment};
+        });
+    } else {
+        entity.lists = [];
+    }
+
+    entity.data = JSON.parse(entity.data);
+
+    return entity;
+}
+
+async function getByIdTx(tx, context, id, withPermissions = true, content = Content.ALL) {
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'view');
+
+    let entity = await rawGetByTx(tx, 'id', id);
+
+    if (content === Content.ALL || content === Content.RSS_ENTRY) {
+        // Return everything
+
+    } else if (content === Content.SETTINGS_WITH_STATS) {
+        delete entity.data.sourceCustom;
+
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
+
+        const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id, true);
+        if (unsentQryGen) {
+            const res = await unsentQryGen(tx).count('* AS subscriptionsToSend').first();
+            entity.subscriptionsToSend = res.subscriptionsToSend;
+        }
+
+        const totalQryGen = await getSubscribersQueryGeneratorTx(tx, id, false);
+        if (totalQryGen) {
+            const res = await totalQryGen(tx).count('* AS subscriptionsTotal').first();
+            entity.subscriptionsTotal = res.subscriptionsTotal;
+        }
+
+    } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+        delete entity.data.sourceCustom;
+
+    } else if (content === Content.ONLY_SOURCE_CUSTOM) {
+        entity = {
+            id: entity.id,
+            send_configuration: entity.send_configuration,
+
+            data: {
+                sourceCustom: entity.data.sourceCustom
+            }
+        };
+    }
+
+    if (withPermissions) {
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'campaign', id);
+    }
+
+    return entity;
+}
+
+async function getById(context, id, withPermissions = true, content = Content.ALL) {
+    return await knex.transaction(async tx => {
+        return await getByIdTx(tx, context, id, withPermissions, content);
+    });
+}
+
+async function _validateAndPreprocess(tx, context, entity, isCreate, content) {
+    if (content === Content.ALL || content === Content.WITHOUT_SOURCE_CUSTOM || content === Content.RSS_ENTRY) {
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        if (isCreate) {
+            enforce(entity.type === CampaignType.REGULAR || entity.type === CampaignType.RSS || entity.type === CampaignType.TRIGGERED ||
+                    (content === Content.RSS_ENTRY && entity.type === CampaignType.RSS_ENTRY),
+                'Unknown campaign type');
+
+            if (entity.source === CampaignSource.TEMPLATE || entity.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
+                await shares.enforceEntityPermissionTx(tx, context, 'template', entity.data.sourceTemplate, 'view');
+            }
+
+            enforce(Number.isInteger(entity.source));
+            enforce(entity.source >= CampaignSource.MIN && entity.source <= CampaignSource.MAX, 'Unknown campaign source');
+        }
+
+        for (const lstSeg of entity.lists) {
+            await shares.enforceEntityPermissionTx(tx, context, 'list', lstSeg.list, 'view');
+
+            if (lstSeg.segment) {
+                // Check that the segment under the list exists
+                await segments.getByIdTx(tx, context, lstSeg.list, lstSeg.segment);
+            }
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', entity.send_configuration, 'viewPublic');
+    }
+}
+
+function convertFileURLs(sourceCustom, fromEntityType, fromEntityId, toEntityType, toEntityId) {
+
+    function convertText(text) {
+        if (text) {
+            const fromUrl = `/files/${fromEntityType}/file/${fromEntityId}`;
+            const toUrl = `/files/${toEntityType}/file/${toEntityId}`;
+
+            const encodedFromUrl = encodeURIComponent(fromUrl);
+            const encodedToUrl = encodeURIComponent(toUrl);
+
+            text = text.split('[URL_BASE]' + fromUrl).join('[URL_BASE]' + toUrl);
+            text = text.split('[SANDBOX_URL_BASE]' + fromUrl).join('[SANDBOX_URL_BASE]' + toUrl);
+            text = text.split('[ENCODED_URL_BASE]' + encodedFromUrl).join('[ENCODED_URL_BASE]' + encodedToUrl);
+            text = text.split('[ENCODED_SANDBOX_URL_BASE]' + encodedFromUrl).join('[ENCODED_SANDBOX_URL_BASE]' + encodedToUrl);
+        }
+
+        return text;
+    }
+
+    sourceCustom.html = convertText(sourceCustom.html);
+    sourceCustom.text = convertText(sourceCustom.text);
+
+    if (sourceCustom.type === 'mosaico' || sourceCustom.type === 'mosaicoWithFsTemplate') {
+        sourceCustom.data.model = convertText(sourceCustom.data.model);
+        sourceCustom.data.model = convertText(sourceCustom.data.model);
+        sourceCustom.data.metadata = convertText(sourceCustom.data.metadata);
+    }
+}
+
+async function _createTx(tx, context, entity, content) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createCampaign');
+
+        let copyFilesFrom = null;
+        if (entity.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
+            copyFilesFrom = {
+                entityType: 'template',
+                entityId: entity.data.sourceTemplate
+            };
+
+            const template = await templates.getByIdTx(tx, context, entity.data.sourceTemplate, false);
+
+            entity.data.sourceCustom = {
+                type: template.type,
+                data: template.data,
+                html: template.html,
+                text: template.text
+            };
+
+        } else if (entity.source === CampaignSource.CUSTOM_FROM_CAMPAIGN) {
+            copyFilesFrom = {
+                entityType: 'campaign',
+                entityId: entity.data.sourceCampaign
+            };
+
+            const sourceCampaign = await getByIdTx(tx, context, entity.data.sourceCampaign, false);
+            enforce(sourceCampaign.source === CampaignSource.CUSTOM || sourceCampaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE || sourceCampaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN, 'Incorrect source type of the source campaign.');
+
+            entity.data.sourceCustom = sourceCampaign.data.sourceCustom;
+        }
+
+        await _validateAndPreprocess(tx, context, entity, true, content);
+
+        const filteredEntity = filterObject(entity, entity.type === CampaignType.RSS_ENTRY ? allowedKeysCreateRssEntry : allowedKeysCreate);
+        filteredEntity.cid = shortid.generate();
+
+        const data = filteredEntity.data;
+
+        filteredEntity.data = JSON.stringify(filteredEntity.data);
+
+        if (filteredEntity.type === CampaignType.RSS || filteredEntity.type === CampaignType.TRIGGERED) {
+            filteredEntity.status = CampaignStatus.ACTIVE;
+        } else if (filteredEntity.type === CampaignType.RSS_ENTRY) {
+            filteredEntity.status = CampaignStatus.SCHEDULED;
+        } else {
+            filteredEntity.status = CampaignStatus.IDLE;
+        }
+
+        const ids = await tx('campaigns').insert(filteredEntity);
+        const id = ids[0];
+
+        await tx('campaign_lists').insert(entity.lists.map(x => ({campaign: id, ...x})));
+
+        if (entity.source === CampaignSource.TEMPLATE) {
+            await tx('template_dep_campaigns').insert({
+               campaign: id,
+               template: entity.data.sourceTemplate
+            });
+        }
+
+        if (filteredEntity.parent) {
+            await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: id, parentId: filteredEntity.parent });
+        } else {
+            await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: id });
+        }
+
+        if (copyFilesFrom) {
+            await files.copyAllTx(tx, context, copyFilesFrom.entityType, 'file', copyFilesFrom.entityId, 'campaign', 'file', id);
+
+            convertFileURLs(data.sourceCustom, copyFilesFrom.entityType, copyFilesFrom.entityId, 'campaign', id);
+            await tx('campaigns')
+                .update({
+                    data: JSON.stringify(data)
+                }).where('id', id);
+        }
+
+        return id;
+    });
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        return await _createTx(tx, context, entity, Content.ALL);
+    });
+}
+
+async function createRssTx(tx, context, entity) {
+    return await _createTx(tx, context, entity, Content.RSS_ENTRY);
+}
+
+async function updateWithConsistencyCheck(context, entity, content) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.id, 'edit');
+
+        const existing = await rawGetByTx(tx, 'id', entity.id);
+
+        const existingHash = hash(existing, content);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, context, entity, false, content);
+
+        let filteredEntity = filterObject(entity, allowedKeysUpdate);
+        if (content === Content.ALL) {
+            await namespaceHelpers.validateMove(context, entity, existing, 'campaign', 'createCampaign', 'delete');
+
+        } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+            filteredEntity.data.sourceCustom = existing.data.sourceCustom;
+            await namespaceHelpers.validateMove(context, filteredEntity, existing, 'campaign', 'createCampaign', 'delete');
+
+        } else if (content === Content.ONLY_SOURCE_CUSTOM) {
+            const data = existing.data;
+            data.sourceCustom = filteredEntity.data.sourceCustom;
+            filteredEntity = {
+                data
+            };
+        }
+
+        if (content === Content.ALL || content === Content.WITHOUT_SOURCE_CUSTOM) {
+            await tx('campaign_lists').where('campaign', entity.id).del();
+            await tx('campaign_lists').insert(entity.lists.map(x => ({campaign: entity.id, ...x})));
+
+            if (existing.source === CampaignSource.TEMPLATE) {
+                await tx('template_dep_campaigns')
+                    .where('campaign', entity.id)
+                    .update('template', entity.data.sourceTemplate);
+            }
+        }
+
+        filteredEntity.data = JSON.stringify(filteredEntity.data);
+        await tx('campaigns').where('id', entity.id).update(filteredEntity);
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: entity.id });
+    });
+}
+
+async function _removeTx(tx, context, id, existing = null) {
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'delete');
+
+    if (!existing) {
+        existing = await tx('campaigns').where('id', id).select(['id', 'status', 'type']).first();
+    }
+
+    if (existing.status === CampaignStatus.SENDING) {
+        return new interoperableErrors.InvalidStateError;
+    }
+
+    enforce(existing.type === CampaignType.REGULAR || existing.type === CampaignType.RSS || existing.type === CampaignType.TRIGGERED, 'This campaign cannot be removed by user.');
+
+    const childCampaigns = await tx('campaigns').where('parent', id).select(['id', 'status', 'type']);
+    for (const childCampaign of childCampaigns) {
+        await _removeTx(tx, contect, childCampaign.id, childCampaign);
+    }
+
+    await files.removeAllTx(tx, context, 'campaign', 'file', id);
+    await files.removeAllTx(tx, context, 'campaign', 'attachment', id);
+
+    await tx('campaign_lists').where('campaign', id).del();
+    await tx('campaign_messages').where('campaign', id).del();
+    await tx('campaign_links').where('campaign', id).del();
+
+    await triggers.removeAllByCampaignIdTx(tx, context, id);
+
+    await tx('template_dep_campaigns')
+        .where('campaign', id)
+        .del();
+
+    await tx('campaigns').where('id', id).del();
+}
+
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await _removeTx(tx, context, id);
+    });
+}
+
+async function enforceSendPermissionTx(tx, context, campaignId) {
+    const campaign = await getByIdTx(tx, context, campaignId, false);
+    const sendConfiguration = await sendConfigurations.getByIdTx(tx, context, campaign.send_configuration, false, false);
+
+    const requiredPermission = getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration);
+
+    await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', campaign.send_configuration, requiredPermission);
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+}
+
+
+// Message API
+
+function getMessageCid(campaignCid, listCid, subscriptionCid) {
+    return [campaignCid, listCid, subscriptionCid].join('.')
+}
+
+async function getMessageByCid(messageCid) {
+    const messageCidElems = messageCid.split('.');
+
+    if (messageCidElems.length !== 3) {
+        return null;
+    }
+
+    const [campaignCid, listCid, subscriptionCid] = messageCidElems;
+
+    await knex.transaction(async tx => {
+        const list = await tx('lists').where('cid', listCid).select('id');
+        const subscrTblName = subscriptions.getSubscriptionTableName(list.id);
+
+        const message = await tx('campaign_messages')
+            .innerJoin('campaigns', 'campaign_messages.campaign', 'campaigns.id')
+            .innerJoin(subscrTblName, subscrTblName + '.id', 'campaign_messages.subscription')
+            .leftJoin('segments', 'segment.id', 'campaign_messages.segment') // This is just to make sure that the respective segment still exists or return null if it doesn't
+            .leftJoin('send_configurations', 'send_configurations.id', 'campaign_messages.send_configuration') // This is just to make sure that the respective send_configuration still exists or return null if it doesn't
+            .where(subscrTblName + '.cid', subscriptionCid)
+            .where('campaigns.cid', campaignCid)
+            .select([
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'segments.id AS segment', 'campaign_messages.subscription',
+                'send_configurations.id AS send_configuration', 'campaign_messages.status', 'campaign_messages.response', 'campaign_messages.response_id',
+                'campaign_messages.updated', 'campaign_messages.created', 'send_configurations.verp_hostname AS verp_hostname'
+            ]);
+
+        if (message) {
+            await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+        }
+
+        return message;
+    });
+}
+
+async function getMessageByResponseId(responseId) {
+    await knex.transaction(async tx => {
+        const message = await tx('campaign_messages')
+            .leftJoin('segments', 'segment.id', 'campaign_messages.segment') // This is just to make sure that the respective segment still exists or return null if it doesn't
+            .leftJoin('send_configurations', 'send_configurations.id', 'campaign_messages.send_configuration') // This is just to make sure that the respective send_configuration still exists or return null if it doesn't
+            .where('campaign_messages.response_id', responseId)
+            .select([
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'segments.id AS segment', 'campaign_messages.subscription',
+                'send_configurations.id AS send_configuration', 'campaign_messages.status', 'campaign_messages.response', 'campaign_messages.response_id',
+                'campaign_messages.updated', 'campaign_messages.created', 'send_configurations.verp_hostname AS verp_hostname'
+            ]);
+
+        if (message) {
+            await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+        }
+
+        return message;
+    });
+}
+
+const statusFieldMapping = {
+    [SubscriptionStatus.UNSUBSCRIBED]: 'unsubscribed',
+    [SubscriptionStatus.BOUNCED]: 'bounced',
+    [SubscriptionStatus.COMPLAINED]: 'complained'
+};
+
+async function _changeStatusByMessageTx(tx, context, message, subscriptionStatus) {
+    enforce(subscriptionStatus !== SubscriptionStatus.SUBSCRIBED);
+
+    if (message.status === SubscriptionStatus.SUBSCRIBED) {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+
+        if (!subscriptionStatus in statusFieldMapping) {
+            throw new Error('Unrecognized message status');
+        }
+
+        const statusField = statusFieldMapping[subscriptionStatus];
+
+        if (message.status === SubscriptionStatus.SUBSCRIBED) {
+            await tx('campaigns').increment(statusField, 1).where('id', message.campaign);
+        }
+
+        await tx('campaign_messages')
+            .where('id', message.id)
+            .update({
+                status: subscriptionStatus,
+                updated: knex.fn.now()
+            });
+    }
+}
+
+async function changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaignCid, listId, subscriptionId, subscriptionStatus) {
+    const campaign = await tx('campaigns').where('cid', campaignCid);
+    const message = await tx('campaign_messages')
+        .innerJoin('campaigns', 'campaign_messages.campaign', 'campaigns.id')
+        .where('campaigns.cid', campaignCid)
+        .where({subscription: subscriptionId, list: listId});
+
+    if (!message) {
+        throw new Error('Invalid campaign.')
+    }
+
+    await _changeStatusByMessageTx(tx, context, message, subscriptionStatus);
+}
+
+async function changeStatusByMessage(context, message, subscriptionStatus, updateSubscription) {
+    await knex.transaction(async tx => {
+        if (updateSubscription) {
+            await subscriptions.changeStatusTx(tx, context, message.list, message.subscription, subscriptionStatus);
+        }
+
+        await _changeStatusByMessageTx(tx, context, message, subscriptionStatus);
+    });
+}
+
+async function updateMessageResponse(context, message, response, responseId) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+
+        await tx('campaign_messages').where('id', message.id).update({
+            response,
+            response_id: responseId
+        });
+    });
+}
+
+async function getSubscribersQueryGeneratorTx(tx, campaignId, onlyUnsent) {
+    /*
+    This is supposed to produce queries like this:
+
+    select ... from `campaign_lists` inner join (
+        select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (
+            (select `subscription__2`.`email`, 8 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__2` left join
+                (select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 2)
+                    as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__2`.`id` where `subscription__2`.`status` = 1)
+            UNION ALL
+            (select `subscription__1`.`email`, 9 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__1` left join
+                (select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 1)
+                    as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__1`.`id` where `subscription__1`.`status` = 1)
+        ) as `pending_subscriptions_all` where `sent` = false group by `email`)
+            as `pending_subscriptions` on `campaign_lists`.`id` = `pending_subscriptions`.`campaign_list_id` where `campaign_lists`.`campaign` = '1'
+
+     This was too much for Knex, so we partially construct these queries directly as strings;
+     */
+
+    const subsQrys = [];
+    const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+
+    for (const cpgList of cpgLists) {
+        const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
+        const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
+
+        const sqlQry = knex.from(subsTable)
+            .leftJoin(
+                function () {
+                    return this.from('campaign_messages')
+                        .where('campaign_messages.campaign', campaignId)
+                        .where('campaign_messages.list', cpgList.list)
+                        .as('related_campaign_messages');
+                },
+                'related_campaign_messages.subscription', subsTable + '.id')
+            .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+            .where(function() {
+                addSegmentQuery(this);
+            })
+            .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('related_campaign_messages.id IS NOT NULL AS sent')])
+            .toSQL().toNative();
+
+        subsQrys.push(sqlQry);
+    }
+
+    if (subsQrys.length > 0) {
+        let subsQry;
+        const unsentWhere = onlyUnsent ? ' where `sent` = false' : '';
+
+        if (subsQrys.length === 1) {
+            const subsUnionSql = '(select `email`, `campaign_list_id`, `sent` from (' + subsQrys[0].sql + ') as `pending_subscriptions_all`' +  unsentWhere + ') as `pending_subscriptions`'
+            subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
+
+        } else {
+            const subsUnionSql = '(select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (' +
+                subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
+                ') as `pending_subscriptions_all`' +  unsentWhere + ' group by `email`) as `pending_subscriptions`';
+            const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+            subsQry = knex.raw(subsUnionSql, subsUnionBindings);
+        }
+
+        return knx => knx.from('campaign_lists')
+            .where('campaign_lists.campaign', campaignId)
+            .innerJoin(subsQry, 'campaign_lists.id', 'pending_subscriptions.campaign_list_id');
+
+    } else {
+        return null;
+    }
+}
+
+async function _changeStatus(context, campaignId, permittedCurrentStates, newState, invalidStateMessage, scheduled = null) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+
+        const entity = await tx('campaigns').where('id', campaignId).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!permittedCurrentStates.includes(entity.status)) {
+            throw new interoperableErrors.InvalidStateError(invalidStateMessage);
+        }
+
+        await tx('campaigns').where('id', campaignId).update({
+            status: newState,
+            scheduled
+        });
+    });
+
+    senders.scheduleCheck();
+}
+
+
+async function start(context, campaignId, startAt) {
+    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.SCHEDULED, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt);
+}
+
+async function stop(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED], CampaignStatus.PAUSED, 'Cannot stop campaign until it is in SCHEDULED state');
+}
+
+async function reset(context, campaignId) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+
+        const entity = await tx('campaigns').where('id', campaignId).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (entity.status !== CampaignStatus.FINISHED && entity.status !== CampaignStatus.PAUSED) {
+            throw new interoperableErrors.InvalidStateError('Cannot reset campaign until it is FINISHED or PAUSED state');
+        }
+
+        await tx('campaigns').where('id', campaignId).update({
+            status: CampaignStatus.IDLE
+        });
+
+        await tx('campaign_messages').where('campaign', campaignId).del();
+        await tx('campaign_links').where('campaign', campaignId).del();
+    });
+}
+
+async function enable(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.INACTIVE], CampaignStatus.ACTIVE, 'Cannot enable campaign unless it is in INACTIVE state');
+}
+
+async function disable(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.ACTIVE], CampaignStatus.INACTIVE, 'Cannot disable campaign unless it is in ACTIVE state');
+}
+
+
+
+module.exports.Content = Content;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listChildrenDTAjax = listChildrenDTAjax;
+module.exports.listWithContentDTAjax = listWithContentDTAjax;
+module.exports.listOthersWhoseListsAreIncludedDTAjax = listOthersWhoseListsAreIncludedDTAjax;
+module.exports.listTestUsersDTAjax = listTestUsersDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.createRssTx = createRssTx;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.enforceSendPermissionTx = enforceSendPermissionTx;
+
+module.exports.getMessageCid = getMessageCid;
+module.exports.getMessageByCid = getMessageByCid;
+module.exports.getMessageByResponseId = getMessageByResponseId;
+
+module.exports.changeStatusByCampaignCidAndSubscriptionIdTx = changeStatusByCampaignCidAndSubscriptionIdTx;
+module.exports.changeStatusByMessage = changeStatusByMessage;
+module.exports.updateMessageResponse = updateMessageResponse;
+
+module.exports.getSubscribersQueryGeneratorTx = getSubscribersQueryGeneratorTx;
+
+module.exports.start = start;
+module.exports.stop = stop;
+module.exports.reset = reset;
+module.exports.enable = enable;
+module.exports.disable = disable;
+
+module.exports.rawGetByTx = rawGetByTx;
+module.exports.getTrackingSettingsByCidTx = getTrackingSettingsByCidTx;

server/models/confirmations.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const shortid = require('shortid');
+
+async function addConfirmation(listId, action, ip, data) {
+    const cid = shortid.generate();
+    await knex('confirmations').insert({
+        cid,
+        list: listId,
+        action,
+        ip,
+        data: JSON.stringify(data || {})
+    });
+
+    return cid;
+}
+
+/*
+ Atomically retrieves confirmation from the database, removes it from the database and returns it.
+ */
+async function takeConfirmation(cid) {
+    return await knex.transaction(async tx => {
+        const entry = await tx('confirmations').select(['cid', 'list', 'action', 'ip', 'data']).where('cid', cid).first();
+
+        if (!entry) {
+            return false;
+        }
+
+        await tx('confirmations').where('cid', cid).del();
+
+        let data;
+        try {
+            data = JSON.parse(entry.data);
+        } catch (err) {
+            data = {};
+        }
+
+        return {
+            list: entry.list,
+            action: entry.action,
+            ip: entry.ip,
+            data
+        };
+    });
+}
+
+module.exports.addConfirmation = addConfirmation;
+module.exports.takeConfirmation = takeConfirmation;

server/models/fields.js

@@ -0,0 +1,828 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const slugify = require('slugify');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const validators = require('../../shared/validators');
+const shortid = require('shortid');
+const segments = require('./segments');
+const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../../shared/date');
+const { getFieldColumn } = require('../../shared/lists');
+const { cleanupFromPost } = require('../lib/helpers');
+const Handlebars = require('handlebars');
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const { getMergeTagsForBases } = require('../../shared/templates');
+
+
+const allowedKeysCreate = new Set(['name', 'key', 'default_value', 'type', 'group', 'settings']);
+const allowedKeysUpdate = new Set(['name', 'key', 'default_value', 'group', 'settings']);
+const hashKeys = allowedKeysCreate;
+
+const fieldTypes = {};
+
+const Cardinality = {
+    SINGLE: 0,
+    MULTIPLE: 1
+};
+
+function render(template, options) {
+    const renderer = Handlebars.compile(template || '');
+    return renderer(options);
+}
+
+fieldTypes.text = {
+    validate: field => {},
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeText',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.website = {
+    validate: field => {},
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeWebsite',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.longtext = {
+    validate: field => {},
+    addColumn: (table, name) => table.text(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeLongtext',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.gpg = {
+    validate: field => {},
+    addColumn: (table, name) => table.text(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeGpg',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.json = {
+    validate: field => {},
+    addColumn: (table, name) => table.json(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeJson',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => {
+        try {
+            if (value === null || value.trim() === '') {
+                return '';
+            }
+
+            let parsed = JSON.parse(value);
+            if (Array.isArray(parsed)) {
+                parsed = {
+                    values: parsed
+                };
+            }
+            return render(field.settings.renderTemplate, parsed);
+        } catch (err) {
+            return err.message;
+        }
+    }
+};
+
+fieldTypes.number = {
+    validate: field => {},
+    addColumn: (table, name) => table.integer(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeNumber',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => Number(value),
+    render: (field, value) => value
+};
+
+fieldTypes['checkbox-grouped'] = {
+    validate: field => {},
+    indexed: true,
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.MULTIPLE,
+    getHbsType: field => 'typeCheckboxGrouped',
+    render: (field, value) => {
+        const subItems = (value || []).map(col => field.groupedOptions[col].name);
+
+        if (field.settings.groupTemplate) {
+            return render(field.settings.groupTemplate, {
+                values: subItems
+            });
+        } else {
+            return subItems.join(', ');
+        }
+    }
+};
+
+fieldTypes['radio-grouped'] = {
+    validate: field => {},
+    indexed: true,
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeRadioGrouped',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes['dropdown-grouped'] = {
+    validate: field => {},
+    indexed: true,
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDropdownGrouped',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes['radio-enum'] = {
+    validate: field => {
+        enforce(field.settings.options, 'Options missing in settings');
+        enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
+    },
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: true,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeRadioEnum',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes['dropdown-enum'] = {
+    validate: field => {
+        enforce(field.settings.options, 'Options missing in settings');
+        enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
+    },
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: true,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDropdownEnum',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes.option = {
+    validate: field => {},
+    addColumn: (table, name) => table.boolean(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    parsePostValue: (field, value) => !(['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0)
+};
+
+fieldTypes['date'] = {
+    validate: field => {
+        enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
+    },
+    addColumn: (table, name) => table.dateTime(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDate' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
+    forHbs: (field, value) => formatDate(field.settings.dateFormat, value),
+    parsePostValue: (field, value) => parseDate(field.settings.dateFormat, value),
+    render: (field, value) => value !== null ? formatDate(field.settings.dateFormat, value) : ''
+};
+
+fieldTypes['birthday'] = {
+    validate: field => {
+        enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
+    },
+    addColumn: (table, name) => table.dateTime(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeBirthday' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
+    forHbs: (field, value) => formatBirthday(field.settings.dateFormat, value),
+    parsePostValue: (field, value) => parseBirthday(field.settings.dateFormat, value),
+    render: (field, value) => value !== null ? formatBirthday(field.settings.dateFormat, value) : ''
+};
+
+const groupedTypes = Object.keys(fieldTypes).filter(key => fieldTypes[key].grouped);
+
+function getFieldType(type) {
+    return fieldTypes[type];
+}
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function getById(context, listId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewFields');
+
+        const entity = await tx('custom_fields').where({list: listId, id}).first();
+
+        entity.settings = JSON.parse(entity.settings);
+
+        const orderFields = {
+            order_list: 'orderListBefore',
+            order_subscribe: 'orderSubscribeBefore',
+            order_manage: 'orderManageBefore'
+        };
+
+        for (const key in orderFields) {
+            if (entity[key] !== null) {
+                const orderIdRow = await tx('custom_fields').where('list', listId).where(key, '>', entity[key]).orderBy(key, 'asc').select(['id']).first();
+                if (orderIdRow) {
+                    entity[orderFields[key]] = orderIdRow.id;
+                } else {
+                    entity[orderFields[key]] = 'end';
+                }
+            } else {
+                entity[orderFields[key]] = 'none';
+            }
+        }
+
+        return entity;
+    });
+}
+
+async function listTx(tx, listId) {
+    return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'key', 'column', 'settings', 'group', 'default_value', 'order_list', 'order_subscribe', 'order_manage']).orderBy(knex.raw('-order_list'), 'desc').orderBy('id', 'asc');
+}
+
+async function list(context, listId) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewFields']);
+        return await listTx(tx, listId);
+    });
+}
+
+async function listGroupedTx(tx, listId) {
+    const flds = await listTx(tx, listId);
+
+    const fldsById = {};
+    for (const fld of flds) {
+        fld.settings = JSON.parse(fld.settings);
+
+        fldsById[fld.id] = fld;
+
+        if (fieldTypes[fld.type].grouped) {
+            fld.settings.options = [];
+            fld.groupedOptions = {};
+        }
+    }
+
+    for (const fld of flds) {
+        if (fld.group) {
+            const group = fldsById[fld.group];
+            group.settings.options.push({ key: fld.column, label: fld.name });
+            group.groupedOptions[fld.column] = fld;
+        }
+    }
+
+    const groupedFlds = flds.filter(fld => !fld.group);
+
+    for (const fld of flds) {
+        delete fld.group;
+    }
+
+    return groupedFlds;
+}
+
+async function listGrouped(context, listId) {
+    return await knex.transaction(async tx => {
+        // It may seem odd why there is not 'viewFields' here. Simply, at this point this function is needed only in managing subscriptions.
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['manageSubscriptions']);
+        return await listGroupedTx(tx, listId);
+    });
+}
+
+async function listByOrderListTx(tx, listId, extraColumns = []) {
+    return await tx('custom_fields').where({list: listId}).whereNotNull('order_list').select(['name', 'type', ...extraColumns]).orderBy('order_list', 'asc');
+}
+
+async function listDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewFields');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('custom_fields')
+
+                // This self join is to provide 'option' fields a reference to their parent grouped field. If the field is not an option, it refers to itself
+                // All this is to show options always below their group parent
+                .innerJoin('custom_fields AS parent_fields', function() {
+                    this.on(function() {
+                        this.on('custom_fields.type', '=', knex.raw('?', ['option']))
+                            .on('custom_fields.group', '=', 'parent_fields.id');
+                    }).orOn(function() {
+                        this.on('custom_fields.type', '<>', knex.raw('?', ['option']))
+                            .on('custom_fields.id', '=', 'parent_fields.id');
+                    });
+                })
+                .where('custom_fields.list', listId),
+            [ 'custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list' ],
+            {
+                orderByBuilder: (builder, orderColumn, orderDir) => {
+                    // We use here parent_fields to keep options always below their parent group
+                    if (orderColumn === 'custom_fields.order_list') {
+                        builder
+                            .orderBy(knex.raw('-parent_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
+                            .orderBy('parent_fields.name', orderDir)
+                            .orderBy(knex.raw('custom_fields.type = "option"'), 'asc')
+                    } else {
+                        const parentColumn = orderColumn.replace(/^custom_fields/, 'parent_fields');
+                        builder
+                            .orderBy(parentColumn, orderDir)
+                            .orderBy('parent_fields.name', orderDir)
+                            .orderBy(knex.raw('custom_fields.type = "option"'), 'asc');
+                    }
+                }
+            }
+        );
+    });
+}
+
+async function listGroupedDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewFields');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('custom_fields')
+                .where('custom_fields.list', listId)
+                .whereIn('custom_fields.type', groupedTypes),
+            ['custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list'],
+            {
+                orderByBuilder: (builder, orderColumn, orderDir) => {
+                    if (orderColumn === 'custom_fields.order_list') {
+                        builder
+                            .orderBy(knex.raw('-custom_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
+                            .orderBy('custom_fields.name', orderDir);
+                    } else {
+                        builder
+                            .orderBy(orderColumn, orderDir)
+                            .orderBy('custom_fields.name', orderDir);
+                    }
+                }
+            }
+        );
+    });
+}
+
+async function serverValidate(context, listId, data) {
+    return await knex.transaction(async tx => {
+        const result = {};
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+        if (data.key) {
+            const existingKeyQuery = tx('custom_fields').where({
+                list: listId,
+                key: data.key
+            });
+
+            if (data.id) {
+                existingKeyQuery.whereNot('id', data.id);
+            }
+
+            const existingKey = await existingKeyQuery.first();
+            result.key = {
+                exists: !!existingKey
+            };
+        }
+
+        return result;
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, entity, isCreate) {
+    enforce(entity.type === 'option' || !entity.group, 'Only option may have a group assigned');
+    enforce(entity.type !== 'option' || entity.group, 'Option must have a group assigned.');
+    enforce(entity.type !== 'option' || (entity.orderListBefore === 'none' && entity.orderSubscribeBefore === 'none' && entity.orderManageBefore === 'none'), 'Option cannot be made visible');
+
+    enforce(!entity.group || await tx('custom_fields').where({list: listId, id: entity.group}).first(), 'Group field does not exist');
+    enforce(entity.name, 'Name must be present');
+
+    const fieldType = fieldTypes[entity.type];
+    enforce(fieldType, 'Unknown field type');
+
+    const validateErrs = fieldType.validate(entity);
+
+    enforce(validators.mergeTagValid(entity.key), 'Merge tag is not valid.');
+
+    const existingWithKeyQuery = tx('custom_fields').where({
+        list: listId,
+        key: entity.key
+    });
+    if (!isCreate) {
+        existingWithKeyQuery.whereNot('id', entity.id);
+    }
+    const existingWithKey = await existingWithKeyQuery.first();
+    if (existingWithKey) {
+        throw new interoperableErrors.DuplicitKeyError();
+    }
+
+    entity.settings = JSON.stringify(entity.settings);
+}
+
+async function _sortIn(tx, listId, entityId, orderListBefore, orderSubscribeBefore, orderManageBefore) {
+    const flds = await tx('custom_fields').where('list', listId).whereNot('id', entityId);
+
+    const order = {};
+    for (const row of flds) {
+        order[row.id] = {
+            order_list: null,
+            order_subscribe: null,
+            order_manage: null
+        };
+    }
+
+    order[entityId] = {
+        order_list: null,
+        order_subscribe: null,
+        order_manage: null
+    };
+
+    function computeOrder(fldName, sortInBefore) {
+        flds.sort((x, y) => x[fldName] - y[fldName]);
+        const ids = flds.filter(x => x[fldName] !== null).map(x => x.id);
+
+        let sortedIn = false;
+        let idx = 1;
+        for (const id of ids) {
+            if (sortInBefore === id) {
+                order[entityId][fldName] = idx;
+                sortedIn = true;
+                idx += 1;
+            }
+
+            order[id][fldName] = idx;
+            idx += 1;
+        }
+
+        if (!sortedIn && sortInBefore !== 'none') {
+            order[entityId][fldName] = idx;
+        }
+    }
+
+    computeOrder('order_list', orderListBefore);
+    computeOrder('order_subscribe', orderSubscribeBefore);
+    computeOrder('order_manage', orderManageBefore);
+
+    for (const id in order) {
+        await tx('custom_fields').where({list: listId, id}).update(order[id]);
+    }
+}
+
+async function create(context, listId, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+        await _validateAndPreprocess(tx, listId, entity, true);
+
+        const fieldType = fieldTypes[entity.type];
+
+        let columnName;
+        if (!fieldType.grouped) {
+            columnName = ('custom_' + slugify(entity.name, '_') + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '');
+        }
+
+        const filteredEntity = filterObject(entity, allowedKeysCreate);
+        filteredEntity.list = listId;
+        filteredEntity.column = columnName;
+
+        const ids = await tx('custom_fields').insert(filteredEntity);
+        const id = ids[0];
+
+        await _sortIn(tx, listId, id, entity.orderListBefore, entity.orderSubscribeBefore, entity.orderManageBefore);
+
+        if (columnName) {
+            await knex.schema.table('subscription__' + listId, table => {
+                fieldType.addColumn(table, columnName);
+                if (fieldType.indexed) {
+                    table.index(columnName);
+                }
+            });
+        }
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, listId, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+        const existing = await tx('custom_fields').where({list: listId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.settings = JSON.parse(existing.settings);
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        enforce(entity.type === existing.type, 'Field type cannot be changed');
+        await _validateAndPreprocess(tx, listId, entity, false);
+
+        await tx('custom_fields').where({list: listId, id: entity.id}).update(filterObject(entity, allowedKeysUpdate));
+        await _sortIn(tx, listId, entity.id, entity.orderListBefore, entity.orderSubscribeBefore, entity.orderManageBefore);
+    });
+}
+
+async function removeTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+    const existing = await tx('custom_fields').where({list: listId, id: id}).first();
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    const fieldType = fieldTypes[existing.type];
+
+    await tx('custom_fields').where({list: listId, id}).del();
+
+    if (fieldType.grouped) {
+        await tx('custom_fields').where({list: listId, group: id}).del();
+
+    } else {
+        await knex.schema.table('subscription__' + listId, table => {
+            table.dropColumn(existing.column);
+        });
+
+        await segments.removeRulesByColumnTx(tx, context, listId, existing.column);
+    }
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, listId, id);
+    });
+}
+
+async function removeAllByListIdTx(tx, context, listId) {
+    const entities = await tx('custom_fields').where('list', listId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, listId, entity.id);
+    }
+}
+
+function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes grouped subscription
+    const customFields = [{
+        name: 'Email Address',
+        column: 'email',
+        key: 'EMAIL',
+        typeSubscriptionEmail: true,
+        value: subscription ? subscription.email : '',
+        order_subscribe: -1,
+        order_manage: -1
+    }];
+
+    for (const fld of fieldsGrouped) {
+        const type = fieldTypes[fld.type];
+        const fldCol = getFieldColumn(fld);
+
+        const entry = {
+            name: fld.name,
+            key: fld.key,
+            [type.getHbsType(fld)]: true,
+            order_subscribe: fld.order_subscribe,
+            order_manage: fld.order_manage,
+        };
+
+        if (!type.grouped && !type.enumerated) {
+            // subscription[fldCol] may not exists because we are getting the data from "fromPost"
+            entry.value = (subscription ? type.forHbs(fld, subscription[fldCol]) : null) || '';
+
+        } else if (type.grouped) {
+            const options = [];
+            const value = (subscription ? subscription[fldCol] : null) || (type.cardinality === Cardinality.SINGLE ? null : []);
+
+            for (const optCol in fld.groupedOptions) {
+                const opt = fld.groupedOptions[optCol];
+
+                let isEnabled;
+                if (type.cardinality === Cardinality.SINGLE) {
+                    isEnabled = value === opt.column;
+                } else {
+                    isEnabled = value.includes(opt.column);
+                }
+
+                options.push({
+                    key: opt.key,
+                    name: opt.name,
+                    value: isEnabled
+                });
+            }
+
+            entry.options = options;
+
+        } else if (type.enumerated) {
+            const options = [];
+            const value = (subscription ? subscription[fldCol] : null) || null;
+
+            for (const opt of fld.settings.options) {
+                options.push({
+                    key: opt.key,
+                    name: opt.label,
+                    value: value === opt.key
+                });
+            }
+
+            entry.options = options;
+
+        }
+
+        customFields.push(entry);
+    }
+
+    return customFields;
+}
+
+// Returns an array that can be used for rendering by Handlebars
+async function forHbs(context, listId, subscription) { // assumes grouped subscription
+    const flds = await listGrouped(context, listId);
+    return forHbsWithFieldsGrouped(flds, subscription);
+}
+
+function getMergeTags(fieldsGrouped, subscription, extraTags = {}) { // assumes grouped subscription
+    const mergeTags = {
+        'EMAIL': subscription.email,
+        ...getMergeTagsForBases(getTrustedUrl(), getSandboxUrl(), getPublicUrl()),
+        ...extraTags
+    };
+
+    for (const fld of fieldsGrouped) {
+        const type = fieldTypes[fld.type];
+        const fldCol = getFieldColumn(fld);
+
+        mergeTags[fld.key] = type.render(fld, subscription[fldCol]);
+    }
+
+    return mergeTags;
+}
+
+
+// Converts subscription data received via (1) POST request from subscription form, (2) via subscribe request to API v1 to subscription structure supported by subscriptions model,
+// or (3) from import.
+// If a field is not specified in the POST data, it is also omitted in the returned subscription
+function _fromText(listId, data, flds, isGrouped, keyName, singleCardUsesKeyName) {
+
+    const subscription = {};
+
+    if (isGrouped) {
+        for (const fld of flds) {
+            const fldKey = fld[keyName];
+            if (fldKey && fldKey in data) {
+                const type = fieldTypes[fld.type];
+                const fldCol = getFieldColumn(fld);
+
+                let value = null;
+
+                if (!type.grouped && !type.enumerated) {
+                    value = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
+
+                } else if (type.grouped) {
+                    if (type.cardinality === Cardinality.SINGLE) {
+                        for (const optCol in fld.groupedOptions) {
+                            const opt = fld.groupedOptions[optCol];
+                            const optKey = opt[keyName];
+
+                            // This handles two different formats for grouped dropdowns and radios.
+                            // The first part of the condition handles the POST requests from the subscription form, while the
+                            // second part handles the subscribe request to API v1
+                            if (singleCardUsesKeyName) {
+                                if (data[fldKey] === optKey) {
+                                    value = opt.column
+                                }
+                            } else {
+                                const optType = fieldTypes[opt.type];
+                                const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
+                                if (optValue) {
+                                    value = opt.column
+                                }
+                            }
+                        }
+                    } else {
+                        value = [];
+
+                        for (const optCol in fld.groupedOptions) {
+                            const opt = fld.groupedOptions[optCol];
+                            const optKey = opt[keyName];
+                            const optType = fieldTypes[opt.type];
+                            const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
+
+                            if (optValue) {
+                                value.push(opt.column);
+                            }
+                        }
+                    }
+
+                } else if (type.enumerated) {
+                    value = data[fldKey];
+                }
+
+                subscription[fldCol] = value;
+            }
+        }
+
+    } else {
+        for (const fld of flds) {
+            const fldKey = fld[keyName];
+            if (fldKey && fldKey in data) {
+                const type = fieldTypes[fld.type];
+                const fldCol = getFieldColumn(fld);
+
+                subscription[fldCol] = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
+            }
+        }
+    }
+
+    return subscription;
+}
+
+async function fromPost(context, listId, data) { // assumes grouped subscription and indexation by merge key
+    const flds = await listGrouped(context, listId);
+    return _fromText(listId, data, flds, true, 'key', true);
+}
+
+async function fromAPI(context, listId, data) { // assumes grouped subscription and indexation by merge key
+    const flds = await listGrouped(context, listId);
+    return _fromText(listId, data, flds, true, 'key', false);
+}
+
+function fromImport(listId, flds, data) { // assumes ungrouped subscription and indexation by column
+    return _fromText(listId, data, flds, true, 'column', false);
+}
+
+module.exports.Cardinality = Cardinality;
+module.exports.getFieldType = getFieldType;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.list = list;
+module.exports.listTx = listTx;
+module.exports.listGrouped = listGrouped;
+module.exports.listGroupedTx = listGroupedTx;
+module.exports.listByOrderListTx = listByOrderListTx;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listGroupedDTAjax = listGroupedDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
+module.exports.serverValidate = serverValidate;
+module.exports.forHbs = forHbs;
+module.exports.forHbsWithFieldsGrouped = forHbsWithFieldsGrouped;
+module.exports.fromPost = fromPost;
+module.exports.fromAPI = fromAPI;
+module.exports.fromImport = fromImport;
+module.exports.getMergeTags = getMergeTags;

server/models/files.js

@@ -0,0 +1,342 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const shares = require('./shares');
+const fs = require('fs-extra-promise');
+const path = require('path');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const entitySettings = require('../lib/entity-settings');
+const {getPublicUrl} = require('../lib/urls');
+
+const crypto = require('crypto');
+const bluebird = require('bluebird');
+const cryptoPseudoRandomBytes = bluebird.promisify(crypto.pseudoRandomBytes);
+
+const entityTypes = entitySettings.getEntityTypes();
+
+const filesDir = path.join(__dirname, '..', 'files');
+
+const ReplacementBehavior = entitySettings.ReplacementBehavior;
+
+function enforceTypePermitted(type, subType) {
+    enforce(type in entityTypes && entityTypes[type].files && entityTypes[type].files[subType], `File type ${type}:${subType} does not exist`);
+}
+
+function getFilePath(type, subType, entityId, filename) {
+    return path.join(path.join(filesDir, type, subType, entityId.toString()), filename);
+}
+
+function getFileUrl(context, type, subType, entityId, filename) {
+    return getPublicUrl(`files/${type}/${subType}/${entityId}/${filename}`, context)
+}
+
+function getFilesTable(type, subType) {
+    return entityTypes[type].files[subType].table;
+}
+
+function getFilesPermission(type, subType, operation) {
+    return entityTypes[type].files[subType].permissions[operation];
+}
+
+async function listDTAjax(context, type, subType, entityId, params) {
+    enforceTypePermitted(type, subType);
+    await shares.enforceEntityPermission(context, type, entityId, getFilesPermission(type, subType, 'view'));
+    return await dtHelpers.ajaxList(
+        params,
+        builder => builder.from(getFilesTable(type, subType)).where({entity: entityId}),
+        ['id', 'originalname', 'filename', 'size', 'created']
+    );
+}
+
+async function listTx(tx, context, type, subType, entityId) {
+    enforceTypePermitted(type, subType);
+    await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
+    return await tx(getFilesTable(type, subType)).where({entity: entityId}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc');
+}
+
+async function list(context, type, subType, entityId) {
+    return await knex.transaction(async tx => {
+        return await listTx(tx, context, type, subType, entityId);
+    });
+}
+
+async function getFileById(context, type, subType, id) {
+    enforceTypePermitted(type, subType);
+    const file = await knex.transaction(async tx => {
+        const file = await tx(getFilesTable(type, subType)).where('id', id).first();
+        await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'view'));
+        return file;
+    });
+
+    if (!file) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    return {
+        mimetype: file.mimetype,
+        name: file.originalname,
+        path: getFilePath(type, subType, file.entity, file.filename)
+    };
+}
+
+async function _getFileBy(context, type, subType, entityId, key, value) {
+    enforceTypePermitted(type, subType);
+    const file = await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
+        const file = await tx(getFilesTable(type, subType)).where({entity: entityId, [key]: value}).first();
+        return file;
+    });
+
+    if (!file) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    return {
+        mimetype: file.mimetype,
+        name: file.originalname,
+        path: getFilePath(type, subType, file.entity, file.filename)
+    };
+}
+
+async function getFileByOriginalName(context, type, subType, entityId, name) {
+    return await _getFileBy(context, type, subType, entityId, 'originalname', name)
+}
+
+async function getFileByFilename(context, type, subType, entityId, name) {
+    return await _getFileBy(context, type, subType, entityId, 'filename', name)
+}
+
+async function getFileByUrl(context, url) {
+    const urlPrefix = getPublicUrl('files/', context);
+    if (url.startsWith(urlPrefix)) {
+        const path = url.substring(urlPrefix.length);
+        const pathElem = path.split('/');
+
+        if (pathElem.length !== 4) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const type = pathElem[0];
+        const subType = pathElem[1];
+        const entityId = Number.parseInt(pathElem[2]);
+
+        if (Number.isNaN(entityId)) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const name = pathElem[3];
+
+        return await getFileByFilename(context, type, subType, entityId, name);
+    } else {
+        throw new interoperableErrors.NotFoundError();
+    }
+}
+
+// Adds files to an entity. The source data can be either a file (then it's path is contained in file.path) or in-memory data (then it's content is in file.data).
+async function createFiles(context, type, subType, entityId, files, replacementBehavior, transformResponseFn) {
+    enforceTypePermitted(type, subType);
+    if (files.length == 0) {
+        // No files uploaded
+        return {uploaded: 0};
+    }
+
+    if (!replacementBehavior) {
+        replacementBehavior = entityTypes[type].files[subType].defaultReplacementBehavior;
+    }
+
+    const fileEntities = [];
+    const filesToMove = [];
+    const ignoredFiles = [];
+    const removedFiles = [];
+    const filesRet = [];
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage'));
+
+        const existingNamesRows = await tx(getFilesTable(type, subType)).where('entity', entityId).select(['id', 'filename', 'originalname']);
+
+        const existingNameSet = new Set();
+        for (const row of existingNamesRows) {
+            existingNameSet.add(row.originalname);
+        }
+
+        // The processedNameSet holds originalnames of entries which have been already processed in the upload batch. It prevents uploading two files with the same originalname
+        const processedNameSet = new Set();
+
+
+        // Create entities for files
+        for (const file of files) {
+            const parsedOriginalName = path.parse(file.originalname);
+            let originalName = parsedOriginalName.base;
+
+            if (!file.filename) {
+                // This is taken from multer/storage/disk.js and adapted for async/await
+                file.filename = (await cryptoPseudoRandomBytes(16)).toString('hex');
+            }
+
+            if (replacementBehavior === ReplacementBehavior.RENAME) {
+                let suffix = 1;
+                while (existingNameSet.has(originalName) || processedNameSet.has(originalName)) {
+                    originalName = parsedOriginalName.name + '-' + suffix + parsedOriginalName.ext;
+                    suffix++;
+                }
+            }
+
+            if (replacementBehavior === ReplacementBehavior.NONE && (existingNameSet.has(originalName) || processedNameSet.has(originalName))) {
+                // The file has an original name same as another file in the same upload batch or it has an original name same as another already existing file
+                ignoredFiles.push(file);
+
+            } else {
+                filesToMove.push(file);
+
+                fileEntities.push({
+                    entity: entityId,
+                    filename: file.filename,
+                    originalname: originalName,
+                    mimetype: file.mimetype,
+                    size: file.size
+                });
+
+                const filesRetEntry = {
+                    name: file.filename,
+                    originalName: originalName,
+                    size: file.size,
+                    type: file.mimetype
+                };
+
+                filesRetEntry.url = getFileUrl(context, type, subType, entityId, file.filename);
+
+                if (file.mimetype.startsWith('image/')) {
+                    filesRetEntry.thumbnailUrl = getFileUrl(context, type, subType, entityId, file.filename); // TODO - use smaller thumbnails,
+                }
+
+                filesRet.push(filesRetEntry);
+            }
+
+            processedNameSet.add(originalName);
+        }
+
+        if (replacementBehavior === ReplacementBehavior.REPLACE) {
+            const idsToRemove = [];
+            for (const row of existingNamesRows) {
+                if (processedNameSet.has(row.originalname)) {
+                    removedFiles.push(row);
+                    idsToRemove.push(row.id);
+                }
+            }
+
+            await tx(getFilesTable(type, subType)).where('entity', entityId).whereIn('id', idsToRemove).del();
+        }
+
+        if (fileEntities) {
+            await tx(getFilesTable(type, subType)).insert(fileEntities);
+        }
+    });
+
+    // Move new files from upload directory to files directory
+    for (const file of filesToMove) {
+        const filePath = getFilePath(type, subType, entityId, file.filename);
+
+        if (file.path) {
+            // The names should be unique, so overwrite is disabled
+            // The directory is created if it does not exist
+            // Empty options argument is passed, otherwise fails
+            await fs.moveAsync(file.path, filePath, {});
+        } else if (file.data) {
+            await fs.outputFile(filePath, file.data);
+        }
+    }
+    // Remove replaced files from files directory
+    for (const file of removedFiles) {
+        const filePath = getFilePath(type, subType, entityId, file.filename);
+        await fs.removeAsync(filePath);
+    }
+    // Remove ignored files from upload directory
+    for (const file of ignoredFiles) {
+        if (file.path) {
+            await fs.removeAsync(file.path);
+        }
+    }
+
+    const resp = {
+        uploaded: files.length,
+        added: fileEntities.length - removedFiles.length,
+        replaced: removedFiles.length,
+        ignored: ignoredFiles.length,
+        files: filesRet
+    };
+
+    if (transformResponseFn) {
+        return transformResponseFn(resp);
+    } else {
+        return resp;
+    }
+}
+
+async function removeFile(context, type, subType, id) {
+    enforceTypePermitted(type, subType);
+
+    const file = await knex.transaction(async tx => {
+        const file = await tx(getFilesTable(type, subType)).where('id', id).select('entity', 'filename').first();
+        await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'manage'));
+        await tx(getFilesTable(type, subType)).where('id', id).del();
+        return {filename: file.filename, entity: file.entity};
+    });
+
+    const filePath = getFilePath(type, subType, file.entity, file.filename);
+    await fs.removeAsync(filePath);
+}
+
+async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toType, toSubType, toEntityId) {
+    enforceTypePermitted(fromType, fromSubType);
+    await shares.enforceEntityPermissionTx(tx, context, fromType, fromEntityId, getFilesPermission(fromType, fromSubType, 'view'));
+
+    enforceTypePermitted(toType, toSubType);
+    await shares.enforceEntityPermissionTx(tx, context, toType, toEntityId, getFilesPermission(toType, toSubType, 'manage'));
+
+    const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId});
+    for (const row of rows) {
+        const fromFilePath = getFilePath(fromType, fromSubType, fromEntityId, row.filename);
+        const toFilePath = getFilePath(toType, toSubType, toEntityId, row.filename);
+        await fs.copyAsync(fromFilePath, toFilePath, {});
+
+        delete row.id;
+        row.entity = toEntityId;
+    }
+
+    if (rows.length > 0) {
+        await tx(getFilesTable(toType, toSubType)).insert(rows);
+    }
+}
+
+async function removeAllTx(tx, context, type, subType, entityId) {
+    enforceTypePermitted(type, subType);
+    await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage'));
+
+    const rows = await tx(getFilesTable(type, subType)).where({entity: entityId});
+    for (const row of rows) {
+        const filePath = getFilePath(type, subType, entityId, row.filename);
+        await fs.removeAsync(filePath);
+    }
+
+    await tx(getFilesTable(type, subType)).where('entity', entityId).del();
+}
+
+
+module.exports.filesDir = filesDir;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listTx = listTx;
+module.exports.list = list;
+module.exports.getFileById = getFileById;
+module.exports.getFileByFilename = getFileByFilename;
+module.exports.getFileByUrl = getFileByUrl;
+module.exports.getFileByOriginalName = getFileByOriginalName;
+module.exports.createFiles = createFiles;
+module.exports.removeFile = removeFile;
+module.exports.getFileUrl = getFileUrl;
+module.exports.getFilePath = getFilePath;
+module.exports.copyAllTx = copyAllTx;
+module.exports.removeAllTx = removeAllTx;
+module.exports.ReplacementBehavior = ReplacementBehavior;

server/models/forms.js

@@ -0,0 +1,278 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const bluebird = require('bluebird');
+const fs = require('fs-extra');
+const path = require('path');
+const mjml = require('mjml');
+const _ = require('../lib/translate')._;
+const lists = require('./lists');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const formAllowedKeys = new Set([
+    'name',
+    'description',
+    'layout',
+    'form_input_style',
+    'namespace'
+]);
+
+const allowedFormKeys = new Set([
+    'web_subscribe',
+    'web_confirm_subscription_notice',
+    'mail_confirm_subscription_html',
+    'mail_confirm_subscription_text',
+    'mail_already_subscribed_html',
+    'mail_already_subscribed_text',
+    'web_subscribed_notice',
+    'mail_subscription_confirmed_html',
+    'mail_subscription_confirmed_text',
+    'web_manage',
+    'web_manage_address',
+    'web_updated_notice',
+    'web_unsubscribe',
+    'web_confirm_unsubscription_notice',
+    'mail_confirm_unsubscription_html',
+    'mail_confirm_unsubscription_text',
+    'mail_confirm_address_change_html',
+    'mail_confirm_address_change_text',
+    'web_unsubscribed_notice',
+    'mail_unsubscription_confirmed_html',
+    'mail_unsubscription_confirmed_text',
+    'web_manual_unsubscribe_notice'
+]);
+
+const hashKeys = new Set([...formAllowedKeys, ...allowedFormKeys]);
+
+const allowedKeysServerValidate = new Set(['layout', ...allowedFormKeys]);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'customForm', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('custom_forms')
+            .innerJoin('namespaces', 'namespaces.id', 'custom_forms.namespace'),
+        ['custom_forms.id', 'custom_forms.name', 'custom_forms.description', 'namespaces.name']
+    );
+}
+
+
+async function _getById(tx, id) {
+    const entity = await tx('custom_forms').where('id', id).first();
+
+    if (!entity) {
+        throw interoperableErrors.NotFoundError();
+    }
+
+    const forms = await tx('custom_forms_data').where('form', id).select(['data_key', 'data_value']);
+
+    for (const form of forms) {
+        entity[form.data_key] = form.data_value;
+    }
+
+    return entity;
+}
+
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'customForm', id, 'view');
+        const entity = await _getById(tx, id);
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'customForm', id);
+        return entity;
+    });
+}
+
+
+async function serverValidate(context, data) {
+    const result = {};
+
+    const form = filterObject(data, allowedKeysServerValidate);
+    const errs = checkForMjmlErrors(form);
+
+    for (const key in form) {
+        result[key] = {};
+        if (errs[key]) {
+            result[key].errors = errs[key];
+        }
+    }
+
+    return result;
+}
+
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createCustomForm');
+
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        const form = filterObject(entity, allowedFormKeys);
+        enforce(!Object.keys(checkForMjmlErrors(form)).length, 'Error(s) in form templates');
+
+        const ids = await tx('custom_forms').insert(filterObject(entity, formAllowedKeys));
+        const id = ids[0];
+
+        for (const formKey in form) {
+            await tx('custom_forms_data').insert({
+                form: id,
+                data_key: formKey,
+                data_value: form[formKey]
+            })
+        }
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'customForm', entityId: id });
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'customForm', entity.id, 'edit');
+
+        const existing = await _getById(tx, entity.id);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'customForm', 'createCustomForm', 'delete');
+
+        const form = filterObject(entity, allowedFormKeys);
+        enforce(!Object.keys(checkForMjmlErrors(form)).length, 'Error(s) in form templates');
+
+        await tx('custom_forms').where('id', entity.id).update(filterObject(entity, formAllowedKeys));
+
+        for (const formKey in form) {
+            await tx('custom_forms_data').update({
+                data_value: form[formKey]
+            }).where({
+                form: entity.id,
+                data_key: formKey
+            });
+        }
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'customForm', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'customForm', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            { entityTypeId: 'list', column: 'default_form' }
+        ]);
+
+        await tx('custom_forms_data').where('form', id).del();
+        await tx('custom_forms').where('id', id).del();
+    });
+}
+
+
+// FIXME - add the ability of having multiple language variant of the same custom form
+async function getDefaultCustomFormValues() {
+    const basePath = path.join(__dirname, '..');
+
+    async function getContents(fileName) {
+        try {
+            const template = await fs.readFile(path.join(basePath, fileName), 'utf8');
+        } catch (err) {
+            return false;
+        }
+    }
+
+    const form = {};
+
+    for (const key of allowedFormKeys) {
+        const base = 'views/subscription/' + key.replace(/_/g, '-');
+        if (key.startsWith('mail') || key.startsWith('web')) {
+            form[key] = await getContents(base + '.mjml.hbs') || await getContents(base + '.hbs') || '';
+        }
+    }
+
+    form.layout = await getContents('views/subscription/layout.mjml.hbs') || '';
+    form.form_input_style = await getContents('static/subscription/form-input-style.css') || '@import url(/subscription/form-input-style.css);';
+
+    return form;
+}
+
+
+function checkForMjmlErrors(form) {
+    let testLayout = '<mjml><mj-body><mj-container>{{{body}}}</mj-container></mj-body></mjml>';
+
+    let hasMjmlError = (template, layout = testLayout) => {
+        let source = layout.replace(/\{\{\{body\}\}\}/g, template);
+        let compiled;
+
+        try {
+            compiled = mjml(source);
+        } catch (err) {
+            return err;
+        }
+
+        return compiled.errors;
+    };
+
+
+    const errors = {};
+    for (const key in form) {
+        if (key.startsWith('mail_') || key.startsWith('web_')) {
+            const template = form[key];
+            const errs = hasMjmlError(template);
+
+            const msgs = errs.map(x => x.formattedMessage);
+            if (key === 'mail_confirm_html' && !template.includes('{{confirmUrl}}')) {
+                msgs.push('Missing {{confirmUrl}}');
+            }
+
+            if (msgs.length) {
+                errors[key] = msgs;
+            }
+
+        } else if (key === 'layout') {
+            const layout = form[key];
+            const errs = hasMjmlError('', layout);
+
+            let msgs;
+            if (Array.isArray(errs)) {
+                msgs = errs.map(x => x.formattedMessage)
+            } else {
+                msgs = [ errs.message ];
+            }
+
+            if (!layout.includes('{{{body}}}')) {
+                msgs.push(`{{{body}}} not found`);
+            }
+
+            if (msgs.length) {
+                errors[key] = msgs;
+            }
+        }
+    }
+
+    return errors;
+}
+
+module.exports.listDTAjax = listDTAjax;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.getDefaultCustomFormValues = getDefaultCustomFormValues;
+module.exports.serverValidate = serverValidate;

server/models/import-runs.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+
+async function getById(context, listId, importId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        const entity = await tx('import_runs')
+            .innerJoin('imports', 'import_runs.import', 'imports.id')
+            .where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': id})
+            .select('import_runs.id', 'import_runs.import', 'import_runs.status', 'import_runs.new',
+                'import_runs.failed', 'import_runs.processed', 'import_runs.error', 'import_runs.created', 'import_runs.finished')
+            .first();
+
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, listId, importId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('import_runs')
+                .innerJoin('imports', 'import_runs.import', 'imports.id')
+                .where({'imports.list': listId, 'imports.id': importId})
+                .orderBy('import_runs.id', 'desc'),
+            [ 'import_runs.id', 'import_runs.created', 'import_runs.finished', 'import_runs.status', 'import_runs.processed', 'import_runs.new', 'import_runs.failed']
+        );
+    });
+}
+
+async function listFailedDTAjax(context, listId, importId, importRunId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('import_failed')
+                .innerJoin('import_runs', 'import_failed.run', 'import_runs.id')
+                .innerJoin('imports', 'import_runs.import', 'imports.id')
+                .where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': importRunId})
+                .orderBy('import_failed.source_id', 'asc'),
+            [ 'import_failed.id', 'import_failed.source_id', 'import_failed.email', 'import_failed.reason']
+        );
+    });
+}
+
+
+
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listFailedDTAjax = listFailedDTAjax;

server/models/imports.js

@@ -0,0 +1,250 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const {ImportSource, MappingType, ImportStatus, RunStatus, prepFinished, prepFinishedAndNotInProgress, runInProgress} = require('../../shared/imports');
+const fs = require('fs-extra-promise');
+const path = require('path');
+const importer = require('../lib/importer');
+
+const filesDir = path.join(__dirname, '..', 'files', 'imports');
+
+const allowedKeysCreate = new Set(['name', 'description', 'source', 'settings']);
+const allowedKeysUpdate = new Set(['name', 'description', 'mapping_type', 'mapping']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeysUpdate));
+}
+
+async function getById(context, listId, id, withSampleRow = false) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        entity.settings = JSON.parse(entity.settings);
+        entity.mapping = JSON.parse(entity.mapping);
+
+        if (withSampleRow && prepFinished(entity.status)) {
+            if (entity.source === ImportSource.CSV_FILE) {
+                const importTable = 'import_file__' + id;
+
+                const row = await tx(importTable).first();
+                delete row.id;
+
+                entity.sampleRow = row;
+            }
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('imports')
+                .where('imports.list', listId),
+            [ 'imports.id', 'imports.name', 'imports.description', 'imports.source', 'imports.status', 'imports.last_run' ]
+        );
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, entity, isCreate) {
+    if (isCreate) {
+        enforce(Number.isInteger(entity.source));
+        enforce(entity.source >= ImportSource.MIN && entity.source <= ImportSource.MAX, 'Invalid import source');
+
+        entity.settings = entity.settings || {};
+
+        if (entity.source === ImportSource.CSV_FILE) {
+            entity.settings.csv = entity.settings.csv || {};
+            enforce(entity.settings.csv.delimiter && entity.settings.csv.delimiter.trim(), 'CSV delimiter must not be empty');
+        }
+
+    } else {
+        enforce(Number.isInteger(entity.mapping_type));
+        enforce(entity.mapping_type >= MappingType.MIN && entity.mapping_type <= MappingType.MAX, 'Invalid mapping type');
+
+        entity.mapping = entity.mapping || { settings: {}, fields: {} };
+    }
+}
+
+async function create(context, listId, entity, files) {
+    const res = await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        await _validateAndPreprocess(tx, listId, entity, true);
+
+        if (entity.source === ImportSource.CSV_FILE) {
+            enforce(files.csvFile, 'File must be included');
+            const csvFile = files.csvFile[0];
+            const filePath = path.join(filesDir, csvFile.filename);
+            await fs.moveAsync(csvFile.path, filePath, {});
+
+            entity.settings.csv = {
+                originalname: csvFile.originalname,
+                filename: csvFile.filename,
+                delimiter: entity.settings.csv.delimiter
+            };
+
+            entity.status = ImportStatus.PREP_SCHEDULED;
+        }
+
+        const filteredEntity = filterObject(entity, allowedKeysCreate);
+        filteredEntity.list = listId;
+        filteredEntity.settings = JSON.stringify(filteredEntity.settings);
+
+        filteredEntity.mapping_type = MappingType.BASIC_SUBSCRIBE; // This is not set in the create form. It can be changed in the update form.
+        filteredEntity.mapping = JSON.stringify({});
+
+        const ids = await tx('imports').insert(filteredEntity);
+        const id = ids[0];
+
+        return id;
+    });
+
+    importer.scheduleCheck();
+    return res;
+}
+
+async function updateWithConsistencyCheck(context, listId, entity) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const existing = await tx('imports').where({list: listId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.mapping = JSON.parse(existing.mapping);
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        enforce(prepFinished(existing.status), 'Cannot save updates until preparation is finished');
+
+        await _validateAndPreprocess(tx, listId, entity, false);
+
+        const filteredEntity = filterObject(entity, allowedKeysUpdate);
+        filteredEntity.mapping = JSON.stringify(filteredEntity.mapping);
+
+        await tx('imports').where({list: listId, id: entity.id}).update(filteredEntity);
+    });
+}
+
+async function removeTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+    const existing = await tx('imports').where({list: listId, id: id}).first();
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    existing.settings = JSON.parse(existing.settings);
+
+    const filePath = path.join(filesDir, existing.settings.csv.filename);
+    await fs.removeAsync(filePath);
+
+    const importTable = 'import_file__' + id;
+    await knex.schema.dropTableIfExists(importTable);
+
+    await tx('import_failed').whereIn('run', function() {this.from('import_runs').select('id').where('import', id)}).del();
+    await tx('import_runs').where('import', id).del();
+    await tx('imports').where({list: listId, id}).del();
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, listId, id);
+    });
+}
+
+async function removeAllByListIdTx(tx, context, listId) {
+    const entities = await tx('imports').where('list', listId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, listId, entity.id);
+    }
+}
+
+async function start(context, listId, id) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!prepFinishedAndNotInProgress(entity.status)) {
+            throw new interoperableErrors.InvalidStateError('Cannot start until preparation or run is finished');
+        }
+
+        await tx('imports').where({list: listId, id}).update({
+            status: ImportStatus.RUN_SCHEDULED
+        });
+
+        await tx('import_runs').insert({
+            import: id,
+            status: RunStatus.SCHEDULED,
+            mapping: entity.mapping
+        });
+    });
+
+    importer.scheduleCheck();
+}
+
+async function stop(context, listId, id) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!runInProgress(entity.status)) {
+            throw new interoperableErrors.InvalidStateError('No import is currently running');
+        }
+
+        await tx('imports').where({list: listId, id}).update({
+            status: ImportStatus.RUN_STOPPING
+        });
+
+        await tx('import_runs').where('import', id).whereIn('status', [RunStatus.SCHEDULED, RunStatus.RUNNING]).update({
+            status: RunStatus.STOPPING
+        });
+    });
+
+    importer.scheduleCheck();
+}
+
+
+module.exports.filesDir = filesDir;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
+module.exports.start = start;
+module.exports.stop = stop;

server/models/links.js

@@ -0,0 +1,180 @@
+'use strict';
+
+const log = require('../lib/log');
+const knex = require('../lib/knex');
+const dtHelpers = require('../lib/dt-helpers');
+const shares = require('./shares');
+const campaigns = require('./campaigns');
+const lists = require('./lists');
+const subscriptions = require('./subscriptions');
+const contextHelpers = require('../lib/context-helpers');
+const geoip = require('geoip-ultralight');
+const uaParser = require('device');
+const he = require('he');
+const { enforce } = require('../lib/helpers');
+const { getPublicUrl } = require('../lib/urls');
+const tools = require('../lib/tools');
+
+const LinkId = {
+    OPEN: -1,
+    GENERAL_CLICK: 0
+};
+
+async function resolve(linkCid) {
+    return await knex('links').where('cid', linkCid).select(['id', 'url']).first();
+}
+
+async function countLink(remoteIp, userAgent, campaignCid, listCid, subscriptionCid, linkId) {
+    await knex.transaction(async tx => {
+        const list = await lists.getByCidTx(tx, contextHelpers.getAdminContext(), listCid);
+        const campaign = await campaigns.getTrackingSettingsByCidTx(tx, campaignCid);
+        const subscription = await subscriptions.getByCidTx(tx, contextHelpers.getAdminContext(), list.id, subscriptionCid);
+
+        const country = geoip.lookupCountry(remoteIp) || null;
+        const device = uaParser(userAgent, { unknownUserAgentDeviceType: 'desktop', emptyUserAgentDeviceType: 'desktop' });
+        const now = new Date();
+
+        const _countLink = async (clickLinkId, incrementOnDup) => {
+            try {
+                const campaignLinksQry = knex('campaign_links')
+                    .insert({
+                        campaign: campaign.id,
+                        list: list.id,
+                        subscription: subscription.id,
+                        link: linkId,
+                        ip: remoteIp,
+                        device_type: device.type,
+                        country
+                    }).toSQL();
+
+                const campaignLinksQryResult = await tx.raw(campaignLinksQry.sql + (incrementOnDup ? ' ON DUPLICATE KEY UPDATE `count`=`count`+1' : ''), campaignLinksQry.bindings);
+
+                if (campaignLinksQryResult.affectedRows > 1) { // When using DUPLICATE KEY UPDATE, this means that the entry was already there
+                    return false;
+                }
+
+                return true;
+
+            } catch (err) {
+                if (err.code === 'ER_DUP_ENTRY') {
+                    return false;
+                }
+
+                throw err;
+            }
+        };
+
+
+
+        // Update opened and click timestamps
+        const latestUpdates = {};
+
+        if (!campaign.click_tracking_disabled && linkId > LinkId.GENERAL_CLICK) {
+            latestUpdates.latest_click = now;
+        }
+
+        if (!campaign.open_tracking_disabled) {
+            latestUpdates.latest_open = now;
+        }
+
+        if (latestUpdates.latest_click || latestUpdates.latest_open) {
+            await tx(subscriptions.getSubscriptionTableName(list.id)).update(latestUpdates).where('id', subscription.id);
+        }
+
+
+        // Update clicks
+        if (linkId > LinkId.GENERAL_CLICK && !campaign.click_tracking_disabled) {
+            if (await _countLink(linkId, true)) {
+                if (await _countLink(LinkId.GENERAL_CLICK, false)) {
+                    await tx('campaigns').increment('clicks').where('id', campaign.id);
+                }
+            }
+        }
+
+
+        // Update opens. We count a click as an open too.
+        if (!campaign.open_tracking_disabled) {
+            if (await _countLink(LinkId.OPEN, true)) {
+                await tx('campaigns').increment('opened').where('id', campaign.id);
+            }
+        }
+    });
+}
+
+async function addOrGet(campaignId, url) {
+    return await knex.transaction(async tx => {
+        const link = tx('links').select(['id', 'cid']).where({
+            campaign: campaignId,
+            url
+        }).first();
+
+        if (!link) {
+            let cid = shortid.generate();
+
+            const ids = tx('links').insert({
+                campaign: campaignId,
+                cid,
+                url
+            });
+
+            return {
+                id: ids[0],
+                cid
+            };
+        }
+    });
+}
+
+async function updateLinks(campaign, list, subscription, mergeTags, message) {
+    if ((campaign.open_tracking_disabled && campaign.click_tracking_disabled) || !message || !message.trim()) {
+        // tracking is disabled, do not modify the message
+        return message;
+    }
+
+    // insert tracking image
+    if (!campaign.open_tracking_disabled) {
+        let inserted = false;
+        const imgUrl = getPublicUrl(`/links/${campaign.cid}/${list.cid}/${subscription.cid}`);
+        const img = '<img src="' + imgUrl + '" width="1" height="1" alt="mt">';
+        message = message.replace(/<\/body\b/i, match => {
+            inserted = true;
+            return img + match;
+        });
+        if (!inserted) {
+            message = message + img;
+        }
+    }
+
+    if (!campaign.click_tracking_disabled) {
+        const re = /(<a[^>]* href\s*=\s*["']\s*)(http[^"'>\s]+)/gi;
+
+        const urlsToBeReplaced = new Set();
+
+        message.replace(re, (match, prefix, encodedUrl) => {
+            const url = he.decode(encodedUrl, {isAttributeValue: true});
+            urlsToBeReplaced.add(url);
+        });
+
+        const urls = new Map(); // url -> {id, cid} (as returned by add)
+        for (const url of urlsToBeReplaced) {
+            // url might include variables, need to rewrite those just as we do with message content
+            const expanedUrl = tools.formatMessage(campaign, list, subscription, mergeTags, url);
+            const link = await addOrGet(campaign.id, expanedUrl);
+            urls.set(url, link);
+        }
+
+        message = message.replace(re, (match, prefix, encodedUrl) => {
+            const url = he.decode(encodedUrl, {isAttributeValue: true});
+            const link = urls.get(url);
+            return prefix + (link ? getPublicUrl(`/links/${campaign.cid}/${list.cid}/${subscription.cid}/${link.cid}`) : url);
+        });
+    }
+
+    return message;
+}
+
+module.exports.LinkId = LinkId;
+module.exports.resolve = resolve;
+module.exports.countLink = countLink;
+module.exports.addOrGet = addOrGet;
+module.exports.updateLinks = updateLinks;

server/models/lists.js

@@ -0,0 +1,210 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const fields = require('./fields');
+const segments = require('./segments');
+const imports = require('./imports');
+const entitySettings = require('../lib/entity-settings');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const UnsubscriptionMode = require('../../shared/lists').UnsubscriptionMode;
+
+const allowedKeys = new Set(['name', 'description', 'default_form', 'public_subscribe', 'unsubscription_mode', 'contact_email', 'homepage', 'namespace', 'to_name', 'listunsubscribe_disabled']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+
+async function listDTAjax(context, params) {
+    const campaignEntityType = entitySettings.getEntityType('campaign');
+
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace'),
+        ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description', 'namespaces.name',
+            { query: builder =>
+                builder.from('campaigns')
+                    .innerJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
+                    .innerJoin('triggers', 'campaigns.id', 'triggers.campaign')
+                    .innerJoin(campaignEntityType.permissionsTable, 'campaigns.id', `${campaignEntityType.permissionsTable}.entity`)
+                    .whereRaw('campaign_lists.list = lists.id')
+                    .where(`${campaignEntityType.permissionsTable}.operation`, 'viewTriggers')
+                    .count()
+            }
+        ]
+    );
+}
+
+async function listWithSegmentByCampaignDTAjax(context, campaignId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('campaign_lists', 'campaign_lists.list', 'lists.id')
+            .leftJoin('segments', 'segments.id', 'campaign_lists.segment')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace')
+            .where('campaign_lists.campaign', campaignId)
+            .orderBy('campaign_lists.id', 'asc'),
+        ['lists.id', 'lists.name', 'lists.cid', 'namespaces.name', 'segments.name']
+    );
+}
+
+async function getByIdTx(tx, context, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', id, 'view');
+    const entity = await tx('lists').where('id', id).first();
+    return entity;
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        // note that permissions are not obtained here as this methods is used only with synthetic admin context
+        return await getByIdTx(tx, context, id);
+    });
+}
+
+async function getByIdWithListFields(context, id) {
+    return await knex.transaction(async tx => {
+        const entity = await getByIdTx(tx, context, id);
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'list', id);
+        entity.listFields = await fields.listByOrderListTx(tx, id);
+        return entity;
+    });
+}
+
+async function getByCidTx(tx, context, cid) {
+    const entity = await tx('lists').where('cid', cid).first();
+    if (!entity) {
+        shares.throwPermissionDenied();
+    }
+
+    await shares.enforceEntityPermissionTx(tx, context, 'list', entity.id, 'view');
+    return entity;
+}
+
+async function getByCid(context, cid) {
+    return await knex.transaction(async tx => {
+        return getByCidTx(tx, context, cid);
+    });
+}
+
+async function _validateAndPreprocess(tx, entity) {
+    await namespaceHelpers.validateEntity(tx, entity);
+    enforce(entity.unsubscription_mode >= UnsubscriptionMode.MIN && entity.unsubscription_mode <= UnsubscriptionMode.MAX, 'Unknown unsubscription mode');
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createList');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.cid = shortid.generate();
+
+        const ids = await tx('lists').insert(filteredEntity);
+        const id = ids[0];
+
+        await knex.schema.raw('CREATE TABLE `subscription__' + id + '` (\n' +
+            '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
+            '  `cid` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
+            '  `email` varchar(255) CHARACTER SET utf8 NOT NULL,\n' +
+            '  `hash_email` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
+            '  `source_email` int(10) unsigned,\n' + // This references imports if the source is an import, 0 means some import in version 1, NULL if the source is via subscription or edit of the subscription
+            '  `opt_in_ip` varchar(100) DEFAULT NULL,\n' +
+            '  `opt_in_country` varchar(2) DEFAULT NULL,\n' +
+            '  `tz` varchar(100) CHARACTER SET ascii DEFAULT NULL,\n' +
+            '  `status` tinyint(4) unsigned NOT NULL DEFAULT \'1\',\n' +
+            '  `is_test` tinyint(4) unsigned NOT NULL DEFAULT \'0\',\n' +
+            '  `status_change` timestamp NULL DEFAULT NULL,\n' +
+            '  `latest_open` timestamp NULL DEFAULT NULL,\n' +
+            '  `latest_click` timestamp NULL DEFAULT NULL,\n' +
+            '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
+            '  PRIMARY KEY (`id`),\n' +
+            '  UNIQUE KEY `email` (`email`),\n' +
+            '  UNIQUE KEY `cid` (`cid`),\n' +
+            '  KEY `status` (`status`),\n' +
+            '  KEY `subscriber_tz` (`tz`),\n' +
+            '  KEY `is_test` (`is_test`),\n' +
+            '  KEY `latest_open` (`latest_open`),\n' +
+            '  KEY `latest_click` (`latest_click`),\n' +
+            '  KEY `created` (`created`)\n' +
+            ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', entity.id, 'edit');
+
+        const existing = await tx('lists').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'list', 'createList', 'delete');
+
+        await tx('lists').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'list', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', id, 'delete');
+
+        await fields.removeAllByListIdTx(tx, context, id);
+        await segments.removeAllByListIdTx(tx, context, id);
+        await imports.removeAllByListIdTx(tx, context, id);
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            {
+                entityTypeId: 'campaign',
+                query: tx => tx('campaign_lists')
+                    .where('campaign_lists.list', id)
+                    .innerJoin('campaigns', 'campaign_lists.campaign', 'campaigns.id')
+                    .select(['campaigns.id', 'campaigns.name'])
+            }
+        ]);
+
+        await tx('lists').where('id', id).del();
+        await knex.schema.dropTableIfExists('subscription__' + id);
+    });
+}
+
+
+module.exports.UnsubscriptionMode = UnsubscriptionMode;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listWithSegmentByCampaignDTAjax = listWithSegmentByCampaignDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.getByIdWithListFields = getByIdWithListFields;
+module.exports.getByCidTx = getByCidTx;
+module.exports.getByCid = getByCid;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/mosaico-templates.js

@@ -0,0 +1,126 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const files = require('./files');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'type', 'data', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'mosaicoTemplate', id, 'view');
+        const entity = await tx('mosaico_templates').where('id', id).first();
+        entity.data = JSON.parse(entity.data);
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'mosaicoTemplate', id);
+        return entity;
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'mosaicoTemplate', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('mosaico_templates').innerJoin('namespaces', 'namespaces.id', 'mosaico_templates.namespace'),
+        [ 'mosaico_templates.id', 'mosaico_templates.name', 'mosaico_templates.description', 'mosaico_templates.type', 'mosaico_templates.created', 'namespaces.name' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, entity) {
+    entity.data = JSON.stringify(entity.data);
+    await namespaceHelpers.validateEntity(tx, entity);
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createMosaicoTemplate');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const ids = await tx('mosaico_templates').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'mosaicoTemplate', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceGlobalPermission(context, 'createJavascriptWithROAccess');
+        await shares.enforceEntityPermissionTx(tx, context, 'mosaicoTemplate', entity.id, 'edit');
+
+        const existing = await tx('mosaico_templates').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.data = JSON.parse(existing.data);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'mosaicoTemplate', 'createMosaicoTemplate', 'delete');
+
+        await tx('mosaico_templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'mosaicoTemplate', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        const deps = [];
+
+        await shares.enforceEntityPermissionTx(tx, context, 'mosaicoTemplate', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            {
+                entityTypeId: 'template',
+                rows: async (tx, limit) => {
+                    const result = [];
+
+                    const tmpls = await tx('templates').where('type', 'mosaico').select(['id', 'name', 'data']);
+                    for (const tmpl of tmpls) {
+                        const data = JSON.parse(tmpl.data);
+                        if (data.mosaicoTemplate === id) {
+                            result.push(tmpl);
+                        }
+
+                        limit -= 1;
+                        if (limit <= 0) break;
+                    }
+
+                    return result;
+                }
+            }
+        ]);
+
+        await files.removeAllTx(tx, context, 'mosaicoTemplate', 'file', id);
+        await files.removeAllTx(tx, context, 'mosaicoTemplate', 'block', id);
+
+        await tx('mosaico_templates').where('id', id).del();
+    });
+}
+
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/namespaces.js

@@ -0,0 +1,192 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const entitySettings = require('../lib/entity-settings');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+
+const allowedKeys = new Set(['name', 'description', 'namespace']);
+
+async function listTree(context) {
+    // FIXME - process permissions
+
+    const entityType = entitySettings.getEntityType('namespace');
+
+    // This builds a forest of namespaces that contains only those namespace that the user has access to
+    // This goes in three steps: 1) tree with all namespaces is built with parent-children links, 2) the namespaces that are not accessible
+    // by the user are pruned out, which potentially transforms the tree to a forest, 3) unneeded attributes (i.e. parent links)
+    // are removed and children are turned to an array are sorted alphabetically by name
+
+    // Build a tree
+    const rows = await knex('namespaces')
+        .leftJoin(entityType.permissionsTable, {
+            [entityType.permissionsTable + '.entity']: 'namespaces.id',
+            [entityType.permissionsTable + '.user']: context.user.id
+        })
+        .groupBy('namespaces.id')
+        .select([
+            'namespaces.id', 'namespaces.name', 'namespaces.description', 'namespaces.namespace',
+            knex.raw(`GROUP_CONCAT(${entityType.permissionsTable + '.operation'} SEPARATOR \';\') as permissions`)
+        ]);
+
+    const entries = {};
+
+    for (let row of rows) {
+        let entry;
+        if (!entries[row.id]) {
+            entry = {
+                children: {}
+            };
+            entries[row.id] = entry;
+        } else {
+            entry = entries[row.id];
+        }
+
+        if (row.namespace) {
+            if (!entries[row.namespace]) {
+                entries[row.namespace] = {
+                    children: {}
+                };
+            }
+
+            entries[row.namespace].children[row.id] = entry;
+            entry.parent = entries[row.namespace];
+        } else {
+            entry.parent = null;
+        }
+
+        entry.key = row.id;
+        entry.title = row.name;
+        entry.description = row.description;
+        entry.permissions = row.permissions ? row.permissions.split(';') : [];
+    }
+
+    // Prune out the inaccessible namespaces
+    for (const entryId in entries) {
+        const entry = entries[entryId];
+
+        if (!entry.permissions.includes('view')) {
+            for (const childId in entry.children) {
+                const child = entry.children[childId];
+                child.parent = entry.parent;
+
+                if (entry.parent) {
+                    entry.parent.children[childId] = child;
+                }
+            }
+
+            if (entry.parent) {
+                delete entry.parent.children[entryId];
+            }
+
+            delete entries[entryId];
+        }
+    }
+
+    // Retrieve the roots before we discard the parent link
+    const roots = Object.values(entries).filter(x => x.parent === null);
+
+    // Remove parent link, transform children to an array and sort it
+    for (const entryId in entries) {
+        const entry = entries[entryId];
+
+        entry.children = Object.values(entry.children);
+        entry.children.sort((x, y) => x.title.localeCompare(y.title));
+
+        delete entry.parent;
+    }
+
+    return roots;
+}
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', id, 'view');
+        const entity = await tx('namespaces').where('id', id).first();
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'namespace', id);
+        return entity;
+    });
+}
+
+async function create(context, entity) {
+    enforce(entity.namespace, 'Parent namespace must be set');
+
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createNamespace');
+
+        const ids = await tx('namespaces').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        // We don't have to rebuild all entity types, because no entity can be a child of the namespace at this moment.
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'namespace', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    enforce(entity.id !== 1 || entity.namespace === null, 'Cannot assign a parent to the root namespace.');
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.id, 'edit');
+
+        const existing = await tx('namespaces').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        // namespaceHelpers.validateEntity is not needed here because it is part of the tree traversal check below
+        await namespaceHelpers.validateMove(context, entity, existing, 'namespace', 'createNamespace', 'delete');
+
+        let iter = entity;
+        while (iter.namespace != null) {
+            iter = await tx('namespaces').where('id', iter.namespace).first();
+
+            if (!iter) {
+                throw new interoperableErrors.DependencyNotFoundError();
+            }
+
+            if (iter.id === entity.id) {
+                throw new interoperableErrors.LoopDetectedError();
+            }
+        }
+
+        await tx('namespaces').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx);
+    });
+}
+
+async function remove(context, id) {
+    enforce(id !== 1, 'Cannot delete the root namespace.');
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', id, 'delete');
+
+        const entityTypesWithNamespace = Object.keys(entitySettings.getEntityTypes());
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, entityTypesWithNamespace.map(entityTypeId => ({ entityTypeId: entityTypeId, column: 'namespace' })));
+
+        await tx('namespaces').where('id', id).del();
+    });
+}
+
+module.exports.hash = hash;
+module.exports.listTree = listTree;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/report-templates.js

@@ -0,0 +1,103 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const reports = require('./reports');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', id, 'view');
+        const entity = await tx('report_templates').where('id', id).first();
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'reportTemplate', id);
+        return entity;
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'reportTemplate', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('report_templates').innerJoin('namespaces', 'namespaces.id', 'report_templates.namespace'),
+        [ 'report_templates.id', 'report_templates.name', 'report_templates.description', 'report_templates.created', 'namespaces.name' ]
+    );
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceGlobalPermission(context, 'createJavascriptWithROAccess');
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createReportTemplate');
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        const ids = await tx('report_templates').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'reportTemplate', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceGlobalPermission(context, 'createJavascriptWithROAccess');
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', entity.id, 'edit');
+
+        const existing = await tx('report_templates').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'reportTemplate', 'createReportTemplate', 'delete');
+
+        await tx('report_templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'reportTemplate', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            { entityTypeId: 'report', column: 'report_template' }
+        ]);
+
+        await tx('report_templates').where('id', id).del();
+    });
+}
+
+async function getUserFieldsById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', id, 'view');
+        const entity = await tx('report_templates').select(['user_fields']).where('id', id).first();
+        return JSON.parse(entity.user_fields);
+    });
+}
+
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.getUserFieldsById = getUserFieldsById;

server/models/reports.js

@@ -0,0 +1,208 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const fields = require('./fields');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const reportHelpers = require('../lib/report-helpers');
+const fs = require('fs-extra-promise');
+
+const ReportState = require('../../shared/reports').ReportState;
+
+const allowedKeys = new Set(['name', 'description', 'report_template', 'params', 'namespace']);
+
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getByIdWithTemplate(context, id, withPermissions = true) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'view');
+
+        const entity = await tx('reports')
+            .where('reports.id', id)
+            .innerJoin('report_templates', 'reports.report_template', 'report_templates.id')
+            .select(['reports.id', 'reports.name', 'reports.description', 'reports.report_template', 'reports.params', 'reports.state', 'reports.namespace', 'report_templates.user_fields', 'report_templates.mime_type', 'report_templates.hbs', 'report_templates.js'])
+            .first();
+
+        entity.user_fields = JSON.parse(entity.user_fields);
+        entity.params = JSON.parse(entity.params);
+
+        if (withPermissions) {
+            entity.permissions = await shares.getPermissionsTx(tx, context, 'report', id);
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [
+            { entityTypeId: 'report', requiredOperations: ['view'] },
+            { entityTypeId: 'reportTemplate', requiredOperations: ['view'] }
+        ],
+        params,
+        builder => builder.from('reports')
+            .innerJoin('report_templates', 'reports.report_template', 'report_templates.id')
+            .innerJoin('namespaces', 'namespaces.id', 'reports.namespace'),
+        [
+            'reports.id', 'reports.name', 'report_templates.name', 'reports.description',
+            'reports.last_run', 'namespaces.name', 'reports.state', 'report_templates.mime_type'
+        ]
+    );
+}
+
+async function create(context, entity) {
+    let id;
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createReport');
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', entity.report_template, 'execute');
+
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        entity.params = JSON.stringify(entity.params);
+
+        const ids = await tx('reports').insert(filterObject(entity, allowedKeys));
+        id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'report', entityId: id });
+    });
+
+    const reportProcessor = require('../lib/report-processor');
+    await reportProcessor.start(id);
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'report', entity.id, 'edit');
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', entity.report_template, 'execute');
+
+        const existing = await tx('reports').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.params = JSON.parse(existing.params);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'report', 'createReport', 'delete');
+
+        entity.params = JSON.stringify(entity.params);
+
+        const filteredUpdates = filterObject(entity, allowedKeys);
+        filteredUpdates.state = ReportState.SCHEDULED;
+
+        await tx('reports').where('id', entity.id).update(filteredUpdates);
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'report', entityId: entity.id });
+    });
+
+    // This require is here to avoid cyclic dependency
+    const reportProcessor = require('../lib/report-processor');
+    await reportProcessor.start(entity.id);
+}
+
+async function removeTx(tx, context, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'delete');
+
+    const report = tx('reports').where('id', id).first();
+
+    await fs.removeAsync(reportHelpers.getReportContentFile(report));
+    await fs.removeAsync(reportHelpers.getReportOutputFile(report));
+
+    await tx('reports').where('id', id).del();
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, id);
+    });
+}
+
+async function updateFields(id, fields) {
+    return await knex('reports').where('id', id).update(fields);
+}
+
+async function listByState(state, limit) {
+    return await knex('reports').where('state', state).limit(limit);
+}
+
+async function bulkChangeState(oldState, newState) {
+    return await knex('reports').where('state', oldState).update('state', newState);
+}
+
+
+const campaignFieldsMapping = {
+    tracker_count: 'campaign_links.count',
+    country: 'campaign_links.country',
+    device_type: 'campaign_links.device_type',
+    status: 'campaign_messages.status',
+    first_name: 'subscriptions.first_name',
+    last_name: 'subscriptions.last_name',
+    email: 'subscriptions.email'
+};
+
+async function getCampaignResults(context, campaign, select, extra) {
+    const flds = await fields.list(context, campaign.list);
+
+    const fieldsMapping = Object.assign({}, campaignFieldsMapping);
+    for (const fld of flds) {
+        /* Dropdown and checkbox groups have field.column == null
+           TODO - For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
+        if (fld.column) {
+            fieldsMapping[fld.key.toLowerCase()] = 'subscriptions.' + fld.column;
+        }
+    }
+
+    let selFields = [];
+    for (let idx = 0; idx < select.length; idx++) {
+        const item = select[idx];
+        if (item in fieldsMapping) {
+            selFields.push(fieldsMapping[item] + ' AS ' + item);
+        } else if (item === '*') {
+            selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
+        } else {
+            selFields.push(item);
+        }
+    }
+
+    let query = knex(`subscription__${campaign.list} AS subscriptions`)
+        .innerJoin('campaign_messages', 'subscriptions.id', 'campaign_messages.subscription')
+        .leftJoin('campaign_links', 'subscriptions.id', 'campaign_links.subscription')
+        .where('campaign_messages.list', campaign.list)
+        .where('campaign_links.list', campaign.list)
+        .select(selFields);
+
+    if (extra) {
+        query = extra(query);
+    }
+
+    return await query;
+}
+
+
+
+module.exports.ReportState = ReportState;
+module.exports.hash = hash;
+module.exports.getByIdWithTemplate = getByIdWithTemplate;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.updateFields = updateFields;
+module.exports.listByState = listByState;
+module.exports.bulkChangeState = bulkChangeState;
+module.exports.getCampaignResults = getCampaignResults;

server/models/segments.js

@@ -0,0 +1,433 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const { enforce, filterObject } = require('../lib/helpers');
+const hasher = require('node-object-hash')();
+const moment = require('moment');
+const fields = require('./fields');
+const subscriptions = require('./subscriptions');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const { parseDate, parseBirthday, DateFormat } = require('../../shared/date');
+
+const allowedKeys = new Set(['name', 'settings']);
+
+
+
+const predefColumns = [
+    {
+        column: 'email',
+        type: 'text'
+    },
+    {
+        column: 'opt_in_country',
+        type: 'text'
+    },
+    {
+        column: 'created',
+        type: 'date'
+    },
+    {
+        column: 'latest_open',
+        type: 'date'
+    },
+    {
+        column: 'latest_click',
+        type: 'date'
+    }
+];
+
+
+const compositeRuleTypes = {
+    all: {
+        addQuery: (query, rules, addSubQuery) => {
+            for (const rule of rules) {
+                query.where(function() {
+                    addSubQuery(this, rule);
+                });
+            }
+        }
+    },
+    some: {
+        addQuery: (query, rules, addSubQuery) => {
+            for (const rule of rules) {
+                query.orWhere(function() {
+                    addSubQuery(this, rule);
+                });
+            }
+        }
+    },
+    none: {
+        addQuery: (query, rules, addSubQuery) => {
+            for (const rule of rules) {
+                query.whereNot(function() {
+                    addSubQuery(this, rule);
+                });
+            }
+        }
+    },
+};
+
+
+
+const primitiveRuleTypes = {
+    text: {},
+    website: {},
+    number: {},
+    date: {},
+    birthday: {},
+    option: {},
+    'dropdown-enum': {},
+    'radio-enum': {}
+};
+
+
+function stringValueSettings(sqlOperator, allowEmpty) {
+    return {
+        validate: rule => {
+            enforce(typeof rule.value === 'string', 'Invalid value type in rule');
+            enforce(allowEmpty || rule.value, 'Value in rule must not be empty');
+        },
+        addQuery: (subsTableName, query, rule) => query.where(subsTableName + '. ' + rule.column, sqlOperator, rule.value)
+    };
+}
+
+function numberValueSettings(sqlOperator) {
+    return {
+        validate: rule => {
+            enforce(typeof rule.value === 'number', 'Invalid value type in rule');
+        },
+        addQuery: (subsTableName, query, rule) => query.where(subsTableName + '. ' + rule.column, sqlOperator, rule.value)
+    };
+}
+
+function dateValueSettings(thisDaySqlOperator, nextDaySqlOperator) {
+    return {
+        validate: rule => {
+            const date = moment.utc(rule.value);
+            enforce(date.isValid(), 'Invalid date value');
+        },
+        addQuery: (subsTableName, query, rule) => {
+            const thisDay = moment.utc(rule.value).startOf('day');
+            const nextDay = moment(thisDay).add(1, 'days');
+
+            if (thisDaySqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, thisDaySqlOperator, thisDay.toDate())
+            }
+
+            if (nextDaySqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, nextDaySqlOperator, nextDay.toDate());
+            }
+        }
+    };
+}
+
+function dateRelativeValueSettings(todaySqlOperator, tomorrowSqlOperator) {
+    return {
+        validate: rule => {
+            enforce(typeof rule.value === 'number', 'Invalid value type in rule');
+        },
+        addQuery: (subsTableName, query, rule) => {
+            const todayWithOffset = moment.utc().startOf('day').add(rule.value, 'days');
+            const tomorrowWithOffset = moment(todayWithOffset).add(1, 'days');
+
+            if (todaySqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, todaySqlOperator, todayWithOffset.toDate())
+            }
+
+            if (tomorrowSqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, tomorrowSqlOperator, tomorrowWithOffset.toDate());
+            }
+        }
+    };
+}
+
+function optionValueSettings(value) {
+    return {
+        validate: rule => {},
+        addQuery: (subsTableName, query, rule) => query.where(subsTableName + '. ' + rule.column, value)
+    };
+}
+
+
+primitiveRuleTypes.text.eq = stringValueSettings('=', true);
+primitiveRuleTypes.text.like = stringValueSettings('LIKE', true);
+primitiveRuleTypes.text.re = stringValueSettings('REGEXP', true);
+primitiveRuleTypes.text.lt = stringValueSettings('<', false);
+primitiveRuleTypes.text.le = stringValueSettings('<=', false);
+primitiveRuleTypes.text.gt = stringValueSettings('>', false);
+primitiveRuleTypes.text.ge = stringValueSettings('>=', false);
+
+primitiveRuleTypes.website.eq = stringValueSettings('=', true);
+primitiveRuleTypes.website.like = stringValueSettings('LIKE', true);
+primitiveRuleTypes.website.re = stringValueSettings('REGEXP', true);
+
+primitiveRuleTypes.number.eq = numberValueSettings('=');
+primitiveRuleTypes.number.lt = numberValueSettings('<');
+primitiveRuleTypes.number.le = numberValueSettings('<=');
+primitiveRuleTypes.number.gt = numberValueSettings('>');
+primitiveRuleTypes.number.ge = numberValueSettings('>=');
+
+primitiveRuleTypes.date.eq = dateValueSettings('>=', '<');
+primitiveRuleTypes.date.lt = dateValueSettings('<', null);
+primitiveRuleTypes.date.le = dateValueSettings(null, '<');
+primitiveRuleTypes.date.gt = dateValueSettings(null, '>=');
+primitiveRuleTypes.date.ge = dateValueSettings('>=', null);
+
+primitiveRuleTypes.date.eqTodayPlusDays = dateRelativeValueSettings('>=', '<');
+primitiveRuleTypes.date.ltTodayPlusDays = dateRelativeValueSettings('<', null);
+primitiveRuleTypes.date.leTodayPlusDays = dateRelativeValueSettings(null, '<');
+primitiveRuleTypes.date.gtTodayPlusDays = dateRelativeValueSettings(null, '>=');
+primitiveRuleTypes.date.geTodayPlusDays = dateRelativeValueSettings('>=', null);
+
+primitiveRuleTypes.birthday.eq = dateValueSettings('>=', '<');
+primitiveRuleTypes.birthday.lt = dateValueSettings('<', null);
+primitiveRuleTypes.birthday.le = dateValueSettings(null, '<');
+primitiveRuleTypes.birthday.gt = dateValueSettings(null, '>=');
+primitiveRuleTypes.birthday.ge = dateValueSettings('>=', null);
+
+primitiveRuleTypes.option.isTrue = optionValueSettings(true);
+primitiveRuleTypes.option.isFalse = optionValueSettings(false);
+
+primitiveRuleTypes['dropdown-enum'].eq = stringValueSettings('=', true);
+primitiveRuleTypes['dropdown-enum'].like = stringValueSettings('LIKE', true);
+primitiveRuleTypes['dropdown-enum'].re = stringValueSettings('REGEXP', true);
+primitiveRuleTypes['dropdown-enum'].lt = stringValueSettings('<', false);
+primitiveRuleTypes['dropdown-enum'].le = stringValueSettings('<=', false);
+primitiveRuleTypes['dropdown-enum'].gt = stringValueSettings('>', false);
+primitiveRuleTypes['dropdown-enum'].ge = stringValueSettings('>=', false);
+
+primitiveRuleTypes['radio-enum'].eq = stringValueSettings('=', true);
+primitiveRuleTypes['radio-enum'].like = stringValueSettings('LIKE', true);
+primitiveRuleTypes['radio-enum'].re = stringValueSettings('REGEXP', true);
+primitiveRuleTypes['radio-enum'].lt = stringValueSettings('<', false);
+primitiveRuleTypes['radio-enum'].le = stringValueSettings('<=', false);
+primitiveRuleTypes['radio-enum'].gt = stringValueSettings('>', false);
+primitiveRuleTypes['radio-enum'].ge = stringValueSettings('>=', false);
+
+
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function listDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSegments');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('segments')
+                .where('list', listId),
+            ['id', 'name']
+        );
+    });
+}
+
+async function listIdName(context, listId) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSegments']);
+
+        return await tx('segments').select(['id', 'name']).where('list', listId).orderBy('name', 'asc');
+    });
+}
+
+async function getByIdTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSegments');
+    const entity = await tx('segments').where({id, list: listId}).first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    entity.settings = JSON.parse(entity.settings);
+    return entity;
+}
+
+async function getById(context, listId, id) {
+    return await knex.transaction(async tx => {
+        return getByIdTx(tx, context, listId, id);
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, entity, isCreate) {
+    enforce(entity.name, 'Name must be present');
+    enforce(entity.settings, 'Settings must be present');
+    enforce(entity.settings.rootRule, 'Root rule must be present in setting');
+    enforce(entity.settings.rootRule.type in compositeRuleTypes, 'Root rule must be composite');
+
+
+    const flds = await fields.listTx(tx, listId);
+    const allowedFlds = [
+        ...predefColumns,
+        ...flds.filter(fld => fld.type in primitiveRuleTypes)
+    ];
+
+    const fieldsByColumn = {};
+    for (const fld of allowedFlds) {
+        fieldsByColumn[fld.column] = fld;
+    }
+
+    function validateRule(rule) {
+        if (rule.type in compositeRuleTypes) {
+            for (const childRule of rule.rules) {
+                validateRule(childRule);
+            }
+        } else {
+            const colType = fieldsByColumn[rule.column].type;
+            primitiveRuleTypes[colType][rule.type].validate(rule);
+        }
+    }
+
+
+    validateRule(entity.settings.rootRule);
+
+    entity.settings = JSON.stringify(entity.settings);
+}
+
+async function create(context, listId, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+        await _validateAndPreprocess(tx, listId, entity, true);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.list = listId;
+
+        const ids = await tx('segments').insert(filteredEntity);
+        const id = ids[0];
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, listId, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+        const existing = await tx('segments').where({list: listId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.settings = JSON.parse(existing.settings);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, listId, entity, false);
+
+        await tx('segments').where({list: listId, id: entity.id}).update(filterObject(entity, allowedKeys));
+    });
+}
+
+
+async function removeTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+    await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+        {
+            entityTypeId: 'campaign',
+            query: tx => tx('campaign_lists')
+                .where('campaign_lists.segment', id)
+                .innerJoin('campaigns', 'campaign_lists.campaign', 'campaigns.id')
+                .select(['campaigns.id', 'campaigns.name'])
+        }
+    ]);
+
+    // The listId "where" is here to prevent deleting segment of a list for which a user does not have permission
+    await tx('segments').where({list: listId, id}).del();
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, listId, id);
+    });
+}
+
+async function removeAllByListIdTx(tx, context, listId) {
+    const entities = await tx('segments').where('list', listId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, listId, entity.id);
+    }
+}
+
+async function removeRulesByColumnTx(tx, context, listId, column) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+    function pruneChildRules(rule) {
+        if (rule.type in compositeRuleTypes) {
+
+            const newRules = [];
+
+            for (const childRule of rule.rules) {
+                if (childRule.column !== column) {
+                    pruneChildRules(childRule);
+                    newRules.push(childRule);
+                }
+            }
+
+            rule.rules = newRules;
+        }
+    }
+
+    const entities = await tx('segments').where({list: listId});
+    for (const entity of entities) {
+        const settings = JSON.parse(entity.settings);
+
+        pruneChildRules(settings.rootRule);
+
+        await tx('segments').where({list: listId, id: entity.id}).update('settings', JSON.stringify(settings));
+    }
+}
+
+async function getQueryGeneratorTx(tx, listId, id) {
+    const flds = await fields.listTx(tx, listId);
+    const allowedFlds = [
+        ...predefColumns,
+        ...flds.filter(fld => fld.type in primitiveRuleTypes)
+    ];
+
+    const fieldsByColumn = {};
+    for (const fld of allowedFlds) {
+        fieldsByColumn[fld.column] = fld;
+    }
+
+    const entity = await tx('segments').where({id, list: listId}).first();
+    const settings = JSON.parse(entity.settings);
+
+    const subsTableName = subscriptions.getSubscriptionTableName(listId);
+
+    function processRule(query, rule) {
+        if (rule.type in compositeRuleTypes) {
+            compositeRuleTypes[rule.type].addQuery(query, rule.rules, (subQuery, childRule) => {
+                processRule(subQuery, childRule);
+            });
+        } else {
+            const colType = fieldsByColumn[rule.column].type;
+            primitiveRuleTypes[colType][rule.type].addQuery(subsTableName, query, rule);
+        }
+    }
+
+    return query => processRule(query, settings.rootRule);
+} 
+
+// This is to handle circular dependency with fields.js
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listIdName = listIdName;
+module.exports.getById = getById;
+module.exports.getByIdTx = getByIdTx;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
+module.exports.removeRulesByColumnTx = removeRulesByColumnTx;
+module.exports.getQueryGeneratorTx = getQueryGeneratorTx;

server/models/send-configurations.js

@@ -0,0 +1,186 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const {MailerType, getSystemSendConfigurationId} = require('../../shared/send-configurations');
+const contextHelpers = require('../lib/context-helpers');
+const mailers = require('../lib/mailers');
+const senders = require('../lib/senders');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable', 'x_mailer', 'verp_hostname', 'mailer_type', 'mailer_settings', 'namespace']);
+
+const allowedMailerTypes = new Set(Object.values(MailerType));
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'sendConfiguration', requiredOperations: ['viewPublic'] }],
+        params,
+        builder => builder
+            .from('send_configurations')
+            .innerJoin('namespaces', 'namespaces.id', 'send_configurations.namespace'),
+        ['send_configurations.id', 'send_configurations.name', 'send_configurations.cid', 'send_configurations.description', 'send_configurations.mailer_type', 'send_configurations.created', 'namespaces.name']
+    );
+}
+
+async function listWithSendPermissionDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'sendConfiguration', requiredOperations: ['sendWithoutOverrides', 'sendWithAllowedOverrides', 'sendWithAnyOverrides'] }],
+        params,
+        builder => builder
+            .from('send_configurations')
+            .innerJoin('namespaces', 'namespaces.id', 'send_configurations.namespace'),
+        ['send_configurations.id', 'send_configurations.name', 'send_configurations.cid', 'send_configurations.description', 'send_configurations.mailer_type', 'send_configurations.created', 'namespaces.name']
+    );
+}
+
+async function _getByTx(tx, context, key, id, withPermissions, withPrivateData) {
+    let entity;
+
+    if (withPrivateData) {
+        entity = await tx('send_configurations').where(key, id).first();
+
+        if (!entity) {
+            shares.throwPermissionDenied();
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', entity.id, 'viewPrivate');
+
+        entity.mailer_settings = JSON.parse(entity.mailer_settings);
+    } else {
+        entity = await tx('send_configurations').where(key, id).select(
+            ['id', 'name', 'cid', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable']
+        ).first();
+
+        if (!entity) {
+            shares.throwPermissionDenied();
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', entity.id, 'viewPublic');
+    }
+
+    // note that permissions are optional as as this methods may be used with synthetic admin context
+    if (withPermissions) {
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'sendConfiguration', id);
+    }
+
+    return entity;
+
+}
+
+async function getByIdTx(tx, context, id, withPermissions = true, withPrivateData = true) {
+    return await _getByTx(tx, context, 'id', id, withPermissions, withPrivateData);
+}
+
+async function getById(context, id, withPermissions = true, withPrivateData = true) {
+    return await knex.transaction(async tx => {
+        return await getByIdTx(tx, context, id, withPermissions, withPrivateData);
+    });
+}
+
+async function getByCid(context, cid, withPermissions = true, withPrivateData = true) {
+    return await knex.transaction(async tx => {
+        return await _getByTx(tx, context, 'cid', cid, withPermissions, withPrivateData);
+    });
+}
+
+async function _validateAndPreprocess(tx, entity, isCreate) {
+    await namespaceHelpers.validateEntity(tx, entity);
+
+    enforce(allowedMailerTypes.has(entity.mailer_type), 'Unknown mailer type');
+    entity.mailer_settings = JSON.stringify(entity.mailer_settings);
+}
+
+
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createSendConfiguration');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.cid = shortid.generate();
+
+        const ids = await tx('send_configurations').insert(filteredEntity);
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'sendConfiguration', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', entity.id, 'edit');
+
+        const existing = await tx('send_configurations').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.mailer_settings = JSON.parse(existing.mailer_settings);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'sendConfiguration', 'createSendConfiguration', 'delete');
+
+        await tx('send_configurations').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'sendConfiguration', entityId: entity.id });
+    });
+
+    mailers.invalidateMailer(entity.id);
+    senders.reloadConfig(entity.id);
+}
+
+async function remove(context, id) {
+    if (id === getSystemSendConfigurationId()) {
+        shares.throwPermissionDenied();
+    }
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            { entityTypeId: 'campaign', column: 'send_configuration' },
+            { entityTypeId: 'list', column: 'send_configuration' }
+        ]);
+
+        await tx('send_configurations').where('id', id).del();
+    });
+}
+
+async function getSystemSendConfiguration() {
+    return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId(), false);
+}
+
+module.exports.MailerType = MailerType;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listWithSendPermissionDTAjax = listWithSendPermissionDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.getByCid = getByCid;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.getSystemSendConfiguration = getSystemSendConfiguration;

server/models/settings.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { filterObject } = require('../lib/helpers');
+const hasher = require('node-object-hash')();
+const shares = require('./shares');
+
+const allowedKeys = new Set(['adminEmail', 'uaCode', 'shoutout', 'pgpPassphrase', 'pgpPrivateKey', 'defaultHomepage']);
+// defaultHomepage is used as a default to list.homepage - if the list.homepage is not filled in
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function get(context, keyOrKeys) {
+    shares.enforceGlobalPermission(context, 'manageSettings');
+
+    let keys;
+    if (!keyOrKeys) {
+        keys = [...allowedKeys.values()];
+    } else if (!Array.isArray(keyOrKeys)) {
+        keys = [ keys ];
+    } else {
+        keys = keyOrKeys;
+    }
+
+    const rows = await knex('settings').select(['key', 'value']).whereIn('key', keys);
+
+    const settings = {};
+    for (const row of rows) {
+        settings[row.key] = row.value;
+    }
+
+    if (!Array.isArray(keyOrKeys) && keyOrKeys) {
+        return settings[keyOrKeys];
+    } else {
+        return settings;
+    }
+}
+
+async function set(context, data) {
+    shares.enforceGlobalPermission(context, 'manageSettings');
+
+    for (const key in data) {
+        if (allowedKeys.has(key)) {
+            const value = data[key];
+            try {
+                await knex('settings').insert({key, value});
+            } catch (err) {
+                await knex('settings').where('key', key).update('value', value);
+            }
+        }
+    }
+
+    // FIXME - recreate mailers, notify senders to recreate the mailers
+}
+
+module.exports.hash = hash;
+module.exports.get = get;
+module.exports.set = set;
+

server/models/shares.js

@@ -0,0 +1,712 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const config = require('config');
+const { enforce } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const entitySettings = require('../lib/entity-settings');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const log = require('../lib/log');
+const {getGlobalNamespaceId} = require('../../shared/namespaces');
+const {getAdminId} = require('../../shared/users');
+
+// TODO: This would really benefit from some permission cache connected to rebuildPermissions
+// A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions
+
+
+async function listByEntityDTAjax(context, entityTypeId, entityId, params) {
+    return await knex.transaction(async (tx) => {
+        const entityType = entitySettings.getEntityType(entityTypeId);
+        await enforceEntityPermissionTx(tx, context, entityTypeId, entityId, 'share');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from(entityType.sharesTable)
+                .innerJoin('users', entityType.sharesTable + '.user', 'users.id')
+                .innerJoin('generated_role_names', 'generated_role_names.role', 'users.role')
+                .where('generated_role_names.entity_type', entityTypeId)
+                .where(`${entityType.sharesTable}.entity`, entityId),
+            ['users.username', 'users.name', 'generated_role_names.name', 'users.id', entityType.sharesTable + '.auto']
+        );
+    });
+}
+
+async function listByUserDTAjax(context, entityTypeId, userId, params) {
+    return await knex.transaction(async (tx) => {
+        const user = await tx('users').where('id', userId).first();
+        if (!user) {
+            shares.throwPermissionDenied();
+        }
+
+        await enforceEntityPermissionTx(tx, context, 'namespace', user.namespace, 'manageUsers');
+
+        const entityType = entitySettings.getEntityType(entityTypeId);
+
+        return await dtHelpers.ajaxListWithPermissionsTx(
+            tx,
+            context,
+            [{entityTypeId}],
+            params,
+            builder => builder
+                .from(entityType.sharesTable)
+                .innerJoin(entityType.entitiesTable, entityType.sharesTable + '.entity', entityType.entitiesTable + '.id')
+                .innerJoin('generated_role_names', 'generated_role_names.role', entityType.sharesTable + '.role')
+                .where('generated_role_names.entity_type', entityTypeId)
+                .where(entityType.sharesTable + '.user', userId),
+            [entityType.entitiesTable + '.name', 'generated_role_names.name', entityType.entitiesTable + '.id', entityType.sharesTable + '.auto']
+        );
+    });
+}
+
+async function listUnassignedUsersDTAjax(context, entityTypeId, entityId, params) {
+    return await knex.transaction(async (tx) => {
+        const entityType = entitySettings.getEntityType(entityTypeId);
+
+        await enforceEntityPermissionTx(tx, context, entityTypeId, entityId, 'share');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('users')
+                .whereNotExists(function () {
+                    return this
+                        .select('*')
+                        .from(entityType.sharesTable)
+                        .whereRaw(`users.id = ${entityType.sharesTable}.user`)
+                        .andWhere(`${entityType.sharesTable}.entity`, entityId);
+                }),
+            ['users.id', 'users.username', 'users.name']
+        );
+    });
+}
+
+async function listRolesDTAjax(entityTypeId, params) {
+    return await dtHelpers.ajaxList(
+        params,
+        builder => builder
+            .from('generated_role_names')
+            .where({entity_type: entityTypeId}),
+        ['role', 'name', 'description']
+    );
+}
+
+async function assign(context, entityTypeId, entityId, userId, role) {
+    const entityType = entitySettings.getEntityType(entityTypeId);
+
+    await knex.transaction(async tx => {
+        await enforceEntityPermissionTx(tx, context, entityTypeId, entityId, 'share');
+
+        enforce(await tx('users').where('id', userId).select('id').first(), 'Invalid user id');
+
+        const extraColumns = entityType.dependentPermissions ? entityType.dependentPermissions.extraColumns : [];
+        const entity = await tx(entityType.entitiesTable).where('id', entityId).select(['id', ...extraColumns]).first();
+        enforce(entity, 'Invalid entity id');
+
+        if (entityType.dependentPermissions) {
+            enforce(!entityType.dependentPermissions.getParent(entity), 'Cannot share/unshare a dependent entity');
+        }
+
+        const entry = await tx(entityType.sharesTable).where({user: userId, entity: entityId}).select('role').first();
+
+        if (entry) {
+            if (!role) {
+                await tx(entityType.sharesTable).where({user: userId, entity: entityId}).del();
+            } else if (entry.role !== role) {
+                await tx(entityType.sharesTable).where({user: userId, entity: entityId}).update('role', role);
+            }
+        } else {
+            await tx(entityType.sharesTable).insert({
+                user: userId,
+                entity: entityId,
+                role
+            });
+        }
+
+        await tx(entityType.permissionsTable).where({user: userId, entity: entityId}).del();
+        if (entityTypeId === 'namespace') {
+            await rebuildPermissionsTx(tx, {userId});
+        } else if (role) {
+            await rebuildPermissionsTx(tx, { entityTypeId, entityId, userId });
+        }
+    });
+}
+
+async function rebuildPermissionsTx(tx, restriction) {
+    restriction = restriction || {};
+
+    const namespaceEntityType = entitySettings.getEntityType('namespace');
+
+    // Collect entity types we care about
+    let restrictedEntityTypes;
+    if (restriction.entityTypeId) {
+        const entityType = entitySettings.getEntityType(restriction.entityTypeId);
+        restrictedEntityTypes = {
+            [restriction.entityTypeId]: entityType
+        };
+    } else {
+        restrictedEntityTypes = entitySettings.getEntityTypesWithPermissions();
+    }
+
+
+    // To prevent users locking out themselves, we consider user with id 1 to be the admin and always assign it
+    // the admin role. The admin role is a global role that has admin===true
+    // If this behavior is not desired, it is enough to delete the user with id 1.
+    const adminUser = await tx('users').where('id', getAdminId()).first();
+    if (adminUser) {
+        let adminRole;
+        for (const role in config.roles.global) {
+            if (config.roles.global[role].admin) {
+                adminRole = role;
+                break;
+            }
+        }
+
+        if (adminRole) {
+            await tx('users').update('role', adminRole).where('id', getAdminId());
+        }
+    }
+
+
+    // Reset root and own namespace shares as per the user roles
+    const usersWithRoleInOwnNamespaceQuery = tx('users')
+        .leftJoin(namespaceEntityType.sharesTable, {
+            'users.id': `${namespaceEntityType.sharesTable}.user`,
+            'users.namespace': `${namespaceEntityType.sharesTable}.entity`
+        })
+        .select(['users.id', 'users.namespace', 'users.role as userRole', `${namespaceEntityType.sharesTable}.role`]);
+    if (restriction.userId) {
+        usersWithRoleInOwnNamespaceQuery.where('users.id', restriction.userId);
+    }
+    const usersWithRoleInOwnNamespace = await usersWithRoleInOwnNamespaceQuery;
+
+    for (const user of usersWithRoleInOwnNamespace) {
+        const roleConf = config.roles.global[user.userRole];
+
+        if (roleConf) {
+            const desiredRole = roleConf.ownNamespaceRole;
+            if (desiredRole && user.role !== desiredRole) {
+                await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: user.namespace }).del();
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: user.namespace, role: desiredRole, auto: true });
+            }
+        }
+    }
+
+
+    const usersWithRoleInRootNamespaceQuery = tx('users')
+        .leftJoin(namespaceEntityType.sharesTable, {
+            'users.id': `${namespaceEntityType.sharesTable}.user`,
+            [`${namespaceEntityType.sharesTable}.entity`]: getGlobalNamespaceId()
+        })
+        .select(['users.id', 'users.role as userRole', `${namespaceEntityType.sharesTable}.role`]);
+    if (restriction.userId) {
+        usersWithRoleInRootNamespaceQuery.andWhere('users.id', restriction.userId);
+    }
+    const usersWithRoleInRootNamespace = await usersWithRoleInRootNamespaceQuery;
+
+    for (const user of usersWithRoleInRootNamespace) {
+        const roleConf = config.roles.global[user.userRole];
+
+        if (roleConf) {
+            const desiredRole = roleConf.rootNamespaceRole;
+            if (desiredRole && user.role !== desiredRole) {
+                await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: getGlobalNamespaceId() }).del();
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: getGlobalNamespaceId(), role: desiredRole, auto: 1 });
+            }
+        }
+    }
+
+
+
+    // Build the map of all namespaces
+    // nsMap is a map of namespaces - each of the following shape:
+    // .id - id of the namespace
+    // .namespace - id of the parent or null if no parent
+    // .userPermissions - Map userId -> [entityTypeId] -> array of permissions
+    // .transitiveUserPermissions - the same as above, but taking into account transitive permission obtained from namespace parents
+
+    const namespaces = await tx('namespaces').select(['id', 'namespace']);
+
+    const nsMap = new Map();
+    for (const namespace of namespaces) {
+        namespace.userPermissions = new Map();
+        nsMap.set(namespace.id, namespace);
+    }
+
+    // This populates .userPermissions
+    const nsSharesQuery = tx(namespaceEntityType.sharesTable).select(['entity', 'user', 'role']);
+    if (restriction.userId) {
+        nsSharesQuery.where('user', restriction.userId);
+    }
+
+    const nsShares = await nsSharesQuery;
+    for (const nsShare of nsShares) {
+        const ns = nsMap.get(nsShare.entity);
+
+        const userPerms = {};
+        ns.userPermissions.set(nsShare.user, userPerms);
+
+        for (const entityTypeId in restrictedEntityTypes) {
+            if (config.roles.namespace[nsShare.role] &&
+                config.roles.namespace[nsShare.role].children &&
+                config.roles.namespace[nsShare.role].children[entityTypeId]) {
+
+                userPerms[entityTypeId] = new Set(config.roles.namespace[nsShare.role].children[entityTypeId]);
+
+            } else {
+                userPerms[entityTypeId] = new Set();
+            }
+        }
+    }
+
+    // This computes .transitiveUserPermissions
+    for (const ns of nsMap.values()) {
+        ns.transitiveUserPermissions = new Map();
+
+        for (const userPermsPair of ns.userPermissions) {
+            const userPerms = {};
+            ns.transitiveUserPermissions.set(userPermsPair[0], userPerms);
+
+            for (const entityTypeId in restrictedEntityTypes) {
+                userPerms[entityTypeId] = new Set(userPermsPair[1][entityTypeId]);
+            }
+        }
+
+        let parentId = ns.namespace;
+        while (parentId) {
+            const parent = nsMap.get(parentId);
+
+            for (const userPermsPair of parent.userPermissions) {
+                const user = userPermsPair[0];
+
+                if (ns.transitiveUserPermissions.has(user)) {
+                    const userPerms = ns.transitiveUserPermissions.get(user);
+
+                    for (const entityTypeId in restrictedEntityTypes) {
+                        for (const perm of userPermsPair[1][entityTypeId]) {
+                            userPerms[entityTypeId].add(perm);
+                        }
+                    }
+                } else {
+                    const userPerms = {};
+                    ns.transitiveUserPermissions.set(user, userPerms);
+
+                    for (const entityTypeId in restrictedEntityTypes) {
+                        userPerms[entityTypeId] = new Set(userPermsPair[1][entityTypeId]);
+                    }
+                }
+            }
+
+            parentId = parent.namespace;
+        }
+    }
+
+
+    // This reads direct shares from DB, joins each with the permissions from namespaces and stores the permissions into DB
+    for (const entityTypeId in restrictedEntityTypes) {
+        const entityType = restrictedEntityTypes[entityTypeId];
+
+        const expungeQuery = tx(entityType.permissionsTable).del();
+        if (restriction.entityId) {
+            expungeQuery.where('entity', restriction.entityId);
+        }
+        if (restriction.userId) {
+            expungeQuery.where('user', restriction.userId);
+        }
+        await expungeQuery;
+
+        const extraColumns = entityType.dependentPermissions ? entityType.dependentPermissions.extraColumns : [];
+        const entitiesQuery = tx(entityType.entitiesTable).select(['id', 'namespace', ...extraColumns]);
+
+
+        const notToBeInserted = new Set();
+        if (restriction.entityId) {
+            if (restriction.parentId) {
+                notToBeInserted.add(restriction.parentId);
+                entitiesQuery.whereIn('id', [restriction.entityId, restriction.parentId]);
+            } else {
+                entitiesQuery.where('id', restriction.entityId);
+            }
+        }
+        const entities = await entitiesQuery;
+
+        // TODO - process restriction.parentId
+
+        const parentEntities = new Map();
+        let nonChildEntities;
+        if (entityType.dependentPermissions) {
+            nonChildEntities = [];
+
+            for (const entity of entities) {
+                const parent = entityType.dependentPermissions.getParent(entity);
+
+                if (parent) {
+                    let childEntities;
+                    if (parentEntities.has(parent)) {
+                        childEntities = parentEntities.get(parent);
+                    } else {
+                        childEntities = [];
+                        parentEntities.set(parent, childEntities);
+                    }
+
+                    childEntities.push(entity.id);
+                } else {
+                    nonChildEntities.push(entity);
+                }
+            }
+        } else {
+            nonChildEntities = entities;
+        }
+
+
+        for (const entity of nonChildEntities) {
+            const permsPerUser = new Map();
+
+            if (entity.namespace) { // The root namespace has not parent namespace, thus the test
+                const transitiveUserPermissions = nsMap.get(entity.namespace).transitiveUserPermissions;
+                for (const transitivePermsPair of transitiveUserPermissions.entries()) {
+                    permsPerUser.set(transitivePermsPair[0], new Set(transitivePermsPair[1][entityTypeId]));
+                }
+            }
+
+            const directSharesQuery = tx(entityType.sharesTable).select(['user', 'role']).where('entity', entity.id);
+            if (restriction.userId) {
+                directSharesQuery.andWhere('user', restriction.userId);
+            }
+            const directShares = await directSharesQuery;
+
+            for (const share of directShares) {
+                let userPerms;
+                if (permsPerUser.has(share.user)) {
+                    userPerms = permsPerUser.get(share.user);
+                } else {
+                    userPerms = new Set();
+                    permsPerUser.set(share.user, userPerms);
+                }
+
+                if (config.roles[entityTypeId][share.role] &&
+                    config.roles[entityTypeId][share.role].permissions) {
+
+                    for (const perm of config.roles[entityTypeId][share.role].permissions) {
+                        userPerms.add(perm);
+                    }
+                }
+            }
+
+            if (!notToBeInserted.has(entity.id)) {
+                for (const userPermsPair of permsPerUser.entries()) {
+                    const data = [];
+
+                    for (const operation of userPermsPair[1]) {
+                        data.push({user: userPermsPair[0], entity: entity.id, operation});
+                    }
+
+                    if (data.length > 0) {
+                        await tx(entityType.permissionsTable).insert(data);
+                    }
+                }
+            }
+
+            if (parentEntities.has(entity.id)) {
+                const childEntities = parentEntities.get(entity.id);
+
+                for (const childId of childEntities) {
+                    for (const userPermsPair of permsPerUser.entries()) {
+                        const data = [];
+
+                        for (const operation of userPermsPair[1]) {
+                            if (operation !== 'share') {
+                                data.push({user: userPermsPair[0], entity: childId, operation});
+                            }
+                        }
+
+                        if (data.length > 0) {
+                            await tx(entityType.permissionsTable).insert(data);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+async function rebuildPermissions(restriction) {
+    await knex.transaction(async tx => {
+        await rebuildPermissionsTx(tx, restriction);
+    });
+}
+
+async function regenerateRoleNamesTable() {
+    await knex.transaction(async tx => {
+        await tx('generated_role_names').del();
+
+        const entityTypeIds = ['global', ...Object.keys(entitySettings.getEntityTypesWithPermissions())];
+
+        for (const entityTypeId of entityTypeIds) {
+            const roles = config.roles[entityTypeId];
+
+            for (const role in roles) {
+                await tx('generated_role_names').insert({
+                    entity_type: entityTypeId,
+                    role,
+                    name: roles[role].name,
+                    description: roles[role].description,
+                });
+            }
+        }
+    });
+}
+
+
+function throwPermissionDenied() {
+    throw new interoperableErrors.PermissionDeniedError('Permission denied');
+}
+
+async function removeDefaultShares(tx, user) {
+    const namespaceEntityType = entitySettings.getEntityType('namespace');
+
+    const roleConf = config.roles.global[user.role];
+
+    if (roleConf) {
+        const desiredRole = roleConf.rootNamespaceRole;
+
+        if (roleConf.ownNamespaceRole) {
+            await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: user.namespace }).del();
+        }
+
+        if (roleConf.rootNamespaceRole) {
+            await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: getGlobalNamespaceId() }).del();
+        }
+    }
+}
+
+function checkGlobalPermission(context, requiredOperations) {
+    if (!context.user) {
+        return false;
+    }
+
+    if (typeof requiredOperations === 'string') {
+        requiredOperations = [ requiredOperations ];
+    }
+
+    if (context.user.restrictedAccessHandler) {
+        const originalRequiredOperations = requiredOperations;
+        const allowedPerms = context.user.restrictedAccessHandler.globalPermissions;
+        if (allowedPerms) {
+            requiredOperations = requiredOperations.filter(perm => allowedPerms.has(perm));
+        } else {
+            requiredOperations = [];
+        }
+        log.verbose('check global permissions with restrictedAccessHandler --  requiredOperations: [' + originalRequiredOperations + '] -> [' + requiredOperations + ']');
+    }
+
+    if (requiredOperations.length === 0) {
+        return false;
+    }
+
+    if (context.user.admin) { // This handles the getAdminContext() case
+        return true;
+    }
+
+    const roleSpec = config.roles.global[context.user.role];
+    let success = false;
+    if (roleSpec) {
+        for (const requiredOperation of requiredOperations) {
+            if (roleSpec.permissions.includes(requiredOperation)) {
+                success = true;
+                break;
+            }
+        }
+    }
+
+    return success;
+}
+
+function enforceGlobalPermission(context, requiredOperations) {
+    if (!checkGlobalPermission(context, requiredOperations)) {
+        throwPermissionDenied();
+    }
+}
+
+async function _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations) {
+    if (!context.user) {
+        return false;
+    }
+
+    const entityType = entitySettings.getEntityType(entityTypeId);
+
+    if (typeof requiredOperations === 'string') {
+        requiredOperations = [ requiredOperations ];
+    }
+
+    if (context.user.restrictedAccessHandler) {
+        const originalRequiredOperations = requiredOperations;
+        if (context.user.restrictedAccessHandler.permissions) {
+            const entityPerms = context.user.restrictedAccessHandler.permissions[entityTypeId];
+
+            if (!entityPerms) {
+                requiredOperations = [];
+            } else if (entityPerms === true) {
+                // no change to require operations
+            } else if (entityPerms instanceof Set) {
+                requiredOperations = requiredOperations.filter(perm => entityPerms.has(perm));
+            } else {
+                const allowedPerms = entityPerms[entityId];
+                if (allowedPerms) {
+                    requiredOperations = requiredOperations.filter(perm => allowedPerms.has(perm));
+                } else {
+                    requiredOperations = [];
+                }
+            }
+        } else {
+            requiredOperations = [];
+        }
+        log.verbose('check permissions with restrictedAccessHandler --  entityTypeId: ' + entityTypeId + '  entityId: ' + entityId + '  requiredOperations: [' + originalRequiredOperations + '] -> [' + requiredOperations + ']');
+    }
+
+    if (requiredOperations.length === 0) {
+        return false;
+    }
+
+    if (context.user.admin) { // This handles the getAdminContext() case. In this case we don't check the permission, but just the existence.
+        const existsQuery = tx(entityType.entitiesTable);
+
+        if (entityId) {
+            existsQuery.where('id', entityId);
+        }
+
+        const exists = await existsQuery.first();
+
+        return !!exists;
+
+    } else {
+        const permsQuery = tx(entityType.permissionsTable)
+            .where('user', context.user.id)
+            .whereIn('operation', requiredOperations);
+
+        if (entityId) {
+            permsQuery.andWhere('entity', entityId);
+        }
+
+        const perms = await permsQuery.first();
+
+        return !!perms;
+    }
+}
+
+async function checkEntityPermission(context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        return false;
+    }
+
+    return await knex.transaction(async tx => {
+        return await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+    });
+}
+
+async function checkEntityPermissionTx(tx, context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        return false;
+    }
+
+    return await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+}
+
+async function checkTypePermission(context, entityTypeId, requiredOperations) {
+    return await knex.transaction(async tx => {
+        return await _checkPermissionTx(tx, context, entityTypeId, null, requiredOperations);
+    });
+}
+
+async function enforceEntityPermission(context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        throwPermissionDenied();
+    }
+    await knex.transaction(async tx => {
+        const result = await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+        if (!result) {
+            log.info(`Denying permission ${entityTypeId}.${entityId} ${requiredOperations}`);
+            throwPermissionDenied();
+        }
+    });
+}
+
+async function enforceEntityPermissionTx(tx, context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        throwPermissionDenied();
+    }
+    const result = await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+    if (!result) {
+        log.info(`Denying permission ${entityTypeId}.${entityId} ${requiredOperations}`);
+        throwPermissionDenied();
+    }
+}
+
+async function enforceTypePermission(context, entityTypeId, requiredOperations) {
+    await knex.transaction(async tx => {
+        const result = await _checkPermissionTx(tx, context, entityTypeId, null, requiredOperations);
+        if (!result) {
+            log.info(`Denying permission ${entityTypeId} ${requiredOperations}`);
+            throwPermissionDenied();
+        }
+    });
+}
+
+async function enforceTypePermissionTx(tx, context, entityTypeId, requiredOperations) {
+    const result = await _checkPermissionTx(tx, context, entityTypeId, null, requiredOperations);
+    if (!result) {
+        log.info(`Denying permission ${entityTypeId} ${requiredOperations}`);
+        throwPermissionDenied();
+    }
+}
+
+function getGlobalPermissions(context) {
+    if (!context.user) {
+        return [];
+    }
+
+    enforce(!context.user.admin, 'getPermissions is not supposed to be called by assumed admin');
+
+    return (config.roles.global[context.user.role] || {}).permissions || [];
+}
+
+async function getPermissionsTx(tx, context, entityTypeId, entityId) {
+    if (!context.user) {
+        return [];
+    }
+
+    enforce(!context.user.admin, 'getPermissions is not supposed to be called by assumed admin');
+
+    const entityType = entitySettings.getEntityType(entityTypeId);
+
+    const rows = await tx(entityType.permissionsTable)
+        .select('operation')
+        .where('entity', entityId)
+        .where('user', context.user.id);
+
+    return rows.map(x => x.operation);
+}
+
+module.exports.listByEntityDTAjax = listByEntityDTAjax;
+module.exports.listByUserDTAjax = listByUserDTAjax;
+module.exports.listUnassignedUsersDTAjax = listUnassignedUsersDTAjax;
+module.exports.listRolesDTAjax = listRolesDTAjax;
+module.exports.assign = assign;
+module.exports.rebuildPermissionsTx = rebuildPermissionsTx;
+module.exports.rebuildPermissions = rebuildPermissions;
+module.exports.removeDefaultShares = removeDefaultShares;
+module.exports.enforceEntityPermission = enforceEntityPermission;
+module.exports.enforceEntityPermissionTx = enforceEntityPermissionTx;
+module.exports.enforceTypePermission = enforceTypePermission;
+module.exports.enforceTypePermissionTx = enforceTypePermissionTx;
+module.exports.checkEntityPermissionTx = checkEntityPermissionTx;
+module.exports.checkEntityPermission = checkEntityPermission;
+module.exports.checkTypePermission = checkTypePermission;
+module.exports.enforceGlobalPermission = enforceGlobalPermission;
+module.exports.checkGlobalPermission = checkGlobalPermission;
+module.exports.throwPermissionDenied = throwPermissionDenied;
+module.exports.regenerateRoleNamesTable = regenerateRoleNamesTable;
+module.exports.getGlobalPermissions = getGlobalPermissions;
+module.exports.getPermissionsTx = getPermissionsTx;

server/models/subscriptions.js

@@ -0,0 +1,826 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const shortid = require('shortid');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const fields = require('./fields');
+const { SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
+const segments = require('./segments');
+const { enforce, filterObject } = require('../lib/helpers');
+const moment = require('moment');
+const { formatDate, formatBirthday } = require('../../shared/date');
+const crypto = require('crypto');
+const campaigns = require('./campaigns');
+const lists = require('./lists');
+
+const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']);
+
+const fieldTypes = {};
+
+const Cardinality = {
+    SINGLE: 0,
+    MULTIPLE: 1
+};
+
+function getOptionsMap(groupedField) {
+    const result = {};
+    for (const opt of groupedField.settings.options) {
+        result[opt.key] = opt.label;
+    }
+
+    return result;
+}
+
+fieldTypes.text = fieldTypes.website = fieldTypes.longtext = fieldTypes.gpg = fieldTypes.number = fieldTypes.json = {
+    afterJSON: (groupedField, entity) => {},
+    listRender: (groupedField, value) => value
+};
+
+fieldTypes['checkbox-grouped'] = {
+    afterJSON: (groupedField, entity) => {},
+    listRender: (groupedField, value) => {
+        const optMap = getOptionsMap(groupedField);
+        return value.map(x => optMap[x]).join(', ');
+    }
+};
+
+fieldTypes['radio-enum'] = fieldTypes['dropdown-enum'] = fieldTypes['radio-grouped'] = fieldTypes['dropdown-grouped'] = {
+    afterJSON: (groupedField, entity) => {},
+    listRender: (groupedField, value) => {
+        const optMap = getOptionsMap(groupedField);
+        return optMap[value];
+    }
+};
+
+fieldTypes.date = {
+    afterJSON: (groupedField, entity) => {
+        const key = getFieldColumn(groupedField);
+        if (key in entity) {
+            entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
+        }
+    },
+    listRender: (groupedField, value) => formatDate(groupedField.settings.dateFormat, value)
+};
+
+fieldTypes.birthday = {
+    afterJSON: (groupedField, entity) => {
+        const key = getFieldColumn(groupedField);
+        if (key in entity) {
+            entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
+        }
+    },
+    listRender: (groupedField, value) => formatBirthday(groupedField.settings.dateFormat, value)
+};
+
+
+
+function getSubscriptionTableName(listId) {
+    return `subscription__${listId}`;
+}
+
+async function getGroupedFieldsMapTx(tx, listId) {
+    const groupedFields = await fields.listGroupedTx(tx, listId);
+    const result = {};
+    for (const fld of groupedFields) {
+        result[getFieldColumn(fld)] = fld;
+    }
+    return result;
+}
+
+function groupSubscription(groupedFieldsMap, entity) {
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+        const fieldType = fields.getFieldType(fld.type);
+
+        if (fieldType.grouped) {
+
+            let value = null;
+
+            if (fieldType.cardinality === fields.Cardinality.SINGLE) {
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+
+                    if (entity[option.column]) {
+                        value = option.column;
+                    }
+
+                    delete entity[option.column];
+                }
+
+            } else {
+                value = [];
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+
+                    if (entity[option.column]) {
+                        value.push(option.column);
+                    }
+
+                    delete entity[option.column];
+                }
+            }
+
+            entity[fldCol] = value;
+
+        } else if (fieldType.enumerated) {
+            // This is enum-xxx type. We just make sure that the options we give out match the field settings.
+            // If the field settings gets changed, there can be discrepancies between the field and the subscription data.
+
+            const allowedKeys = new Set(fld.settings.options.map(x => x.key));
+
+            if (!allowedKeys.has(entity[fldCol])) {
+                entity[fldCol] = null;
+            }
+        }
+    }
+}
+
+function ungroupSubscription(groupedFieldsMap, entity) {
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+
+        const fieldType = fields.getFieldType(fld.type);
+        if (fieldType.grouped) {
+
+            if (fieldType.cardinality === fields.Cardinality.SINGLE) {
+                const value = entity[fldCol];
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+                    entity[option.column] = option.column === value;
+                }
+
+            } else {
+                const values = entity[fldCol] || []; // The default (empty array) is here because create may be called with an entity that has some fields not filled in
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+                    entity[option.column] = values.includes(option.column);
+                }
+            }
+
+            delete entity[fldCol];
+
+        } else if (fieldType.enumerated) {
+            // This is enum-xxx type. We just make sure that the options we give out match the field settings.
+            // If the field settings gets changed, there can be discrepancies between the field and the subscription data.
+
+            const allowedKeys = new Set(fld.settings.options.map(x => x.key));
+
+            if (!allowedKeys.has(entity[fldCol])) {
+                entity[fldCol] = null;
+            }
+        }
+    }
+}
+
+
+function getAllowedKeys(groupedFieldsMap) {
+    return new Set([
+        ...allowedKeysBase,
+        ...Object.keys(groupedFieldsMap)
+    ]);
+}
+
+function hashByAllowedKeys(allowedKeys, entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function hashByList(listId, entity) {
+    return await knex.transaction(async tx => {
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        const allowedKeys = getAllowedKeys(groupedFieldsMap);
+        return hashByAllowedKeys(allowedKeys, entity);
+    });
+}
+
+async function _getByTx(tx, context, listId, key, value, grouped) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+    const entity = await tx(getSubscriptionTableName(listId)).where(key, value).first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+    }
+
+    const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+    if (grouped) {
+        groupSubscription(groupedFieldsMap, entity);
+    }
+
+    return entity;
+}
+
+async function _getBy(context, listId, key, value, grouped) {
+    return await knex.transaction(async tx => {
+        return _getByTx(tx, context, listId, key, value, grouped);
+    });
+}
+
+async function getById(context, listId, id, grouped = true) {
+    return await _getBy(context, listId, 'id', id, grouped);
+}
+
+async function getByEmail(context, listId, email, grouped = true) {
+    return await _getBy(context, listId, 'email', email, grouped);
+}
+
+async function getByCid(context, listId, cid, grouped = true) {
+    return await _getBy(context, listId, 'cid', cid, grouped);
+}
+
+async function getByCidTx(tx, context, listId, cid, grouped = true) {
+    return await _getByTx(tx, context, listId, 'cid', cid, grouped);
+}
+
+async function listDTAjax(context, listId, segmentId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+        const listTable = getSubscriptionTableName(listId);
+
+        // All the data transformation below is to reuse ajaxListTx and groupSubscription methods so as to keep the code DRY
+        // We first construct the columns to contain all which is supposed to be show and extraColumns which contain
+        // everything else that constitutes the subscription.
+        // Then in ajaxList's mapFunc, we construct the entity from the fields ajaxList retrieved and pass it to groupSubscription
+        // to group the fields. Then we copy relevant values form grouped subscription to ajaxList's data which then get
+        // returned to the client. During the copy, we also render the values.
+
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        const listFlds = await fields.listByOrderListTx(tx, listId, ['column', 'id']);
+
+        const columns = [
+            listTable + '.id',
+            listTable + '.cid',
+            listTable + '.email',
+            listTable + '.status',
+            listTable + '.created',
+            { name: 'blacklisted', raw: 'not isnull(blacklist.email)' }
+        ];
+        const extraColumns = [];
+        let listFldIdx = columns.length;
+        const idxMap = {};
+
+        for (const listFld of listFlds) {
+            const fldCol = getFieldColumn(listFld);
+            const fld = groupedFieldsMap[fldCol];
+
+            if (fld.column) {
+                columns.push(listTable + '.' + fld.column);
+            } else {
+                columns.push({
+                    name: listTable + '.' + fldCol,
+                    raw: 0
+                })
+            }
+
+            idxMap[fldCol] = listFldIdx;
+            listFldIdx += 1;
+        }
+
+        for (const fldCol in groupedFieldsMap) {
+            const fld = groupedFieldsMap[fldCol];
+
+            if (fld.column) {
+                if (!(fldCol in idxMap)) {
+                    extraColumns.push(listTable + '.' + fld.column);
+                    idxMap[fldCol] = listFldIdx;
+                    listFldIdx += 1;
+                }
+
+            } else {
+                for (const optionColumn in fld.groupedOptions) {
+                    extraColumns.push(listTable + '.' + optionColumn);
+                    idxMap[optionColumn] = listFldIdx;
+                    listFldIdx += 1;
+                }
+            }
+        }
+
+        const addSegmentQuery = segmentId ? await segments.getQueryGeneratorTx(tx, listId, segmentId) : () => {};
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => {
+                const query = builder
+                    .from(listTable)
+                    .leftOuterJoin('blacklist', listTable + '.email', 'blacklist.email')
+                ;
+                query.where(function() {
+                    addSegmentQuery(this);
+                });
+
+                return query;
+            },
+            columns,
+            {
+                mapFun: data => {
+                    const entity = {};
+                    for (const fldCol in idxMap) {
+                        // This is a bit of hacking. We rely on the fact that if a field has a column, then the column is the field key.
+                        // Then it has the group id with value 0. groupSubscription will be able to process the fields that have a column
+                        // and it will assign values to the fields that don't have a value (i.e. those that currently have the group id and value 0).
+                        entity[fldCol] = data[idxMap[fldCol]];
+                    }
+
+                    groupSubscription(groupedFieldsMap, entity);
+
+                    for (const listFld of listFlds) {
+                        const fldCol = getFieldColumn(listFld);
+                        const fld = groupedFieldsMap[fldCol];
+                        data[idxMap[fldCol]] = fieldTypes[fld.type].listRender(fld, entity[fldCol]);
+                    }
+                },
+
+                extraColumns
+            }
+        );
+    });
+}
+
+async function listTestUsersDTAjax(context, listCid, params) {
+    return await knex.transaction(async tx => {
+        const list = await lists.getByCidTx(tx, context, listCid);
+        await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
+
+        const listTable = getSubscriptionTableName(list.id);
+
+        const columns = [
+            listTable + '.id',
+            listTable + '.cid',
+            listTable + '.email',
+            listTable + '.status',
+            listTable + '.created'
+        ];
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from(listTable)
+                .where('is_test', true),
+            columns,
+            {}
+        );
+    });
+}
+
+async function list(context, listId, grouped = true, offset, limit) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+        const count = await tx(getSubscriptionTableName(listId)).count('* as count').first().count;
+
+        const entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc');
+
+        if (Number.isInteger(offset)) {
+            entitiesQry.offset(offset);
+        }
+
+        if (Number.isInteger(limit)) {
+            entitiesQry.limit(limit);
+        }
+
+        const entities = await entitiesQry;
+
+        if (grouped) {
+            const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+            for (const entity of entities) {
+                groupSubscription(groupedFieldsMap, entity);
+            }
+        }
+
+        return {
+            subscriptions: entities,
+            total: count
+        };
+    });
+}
+
+// Note that this does not do all the work in the transaction. Thus it is prone to fail if the list is deleted in during the run of the function
+async function* listIterator(context, listId, segmentId, grouped = true) {
+    let groupedFieldsMap;
+    let addSegmentQuery;
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+        if (grouped) {
+            groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        }
+
+        addSegmentQuery = segmentId ? await segments.getQueryGeneratorTx(tx, listId, segmentId) : () => {};
+    });
+
+    let lastId = 0;
+
+    while (true) {
+        const entities = await knex(getSubscriptionTableName(listId))
+            .orderBy('id', 'asc')
+            .where('id', '>', lastId)
+            .where(function() {
+                addSegmentQuery(this);
+            })
+            .limit(500);
+
+        if (entities.length > 0) {
+            for (const entity of entities) {
+                if (grouped) {
+                    groupSubscription(groupedFieldsMap, entity);
+                }
+
+                yield entity;
+            }
+
+            lastId = entities[entities.length - 1].id;
+        } else {
+            break;
+        }
+    }
+}
+
+async function serverValidate(context, listId, data) {
+    return await knex.transaction(async tx => {
+        const result = {};
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        if (data.email) {
+            const existingKeyQuery = tx(getSubscriptionTableName(listId)).where('email', data.email);
+
+            if (data.id) {
+                existingKeyQuery.whereNot('id', data.id);
+            }
+
+            const existingKey = await existingKeyQuery.first();
+            result.key = {
+                exists: !!existingKey
+            };
+        }
+
+        return result;
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) {
+    enforce(entity.email, 'Email must be set');
+
+    const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(entity.email));
+
+    if (!isCreate) {
+        existingWithKeyQuery.whereNot('id', entity.id);
+    }
+    const existingWithKey = await existingWithKeyQuery.first();
+    if (existingWithKey) {
+        if (meta && (meta.updateAllowed || (meta.updateOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED))) {
+            meta.update = true;
+            meta.existing = existingWithKey;
+        } else {
+            throw new interoperableErrors.DuplicitEmailError();
+        }
+    } else {
+        // This is here because of the API endpoint, which allows one to submit subscriptions without caring about whether they already exist, what their status is, etc.
+        // The same for import where we need to subscribed only those (existing and new) that have not been unsubscribed already.
+        // In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status to SUBSCRIBED
+
+        if (meta && meta.subscribeIfNoExisting && !entity.status) {
+            entity.status = SubscriptionStatus.SUBSCRIBED;
+        }
+    }
+
+    if ((isCreate && !(meta && meta.update)) || 'status' in entity) {
+        enforce(entity.status >= SubscriptionStatus.MIN && entity.status <= SubscriptionStatus.MAX, 'Invalid status');
+    }
+
+    for (const key in groupedFieldsMap) {
+        const fld = groupedFieldsMap[key];
+
+        fieldTypes[fld.type].afterJSON(fld, entity);
+    }
+}
+
+function hashEmail(email) {
+    return crypto.createHash('sha512').update(email).digest("base64");
+}
+
+function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
+    if ('email' in subscription) {
+        subscription.hash_email = hashEmail(subscription.email);
+        subscription.source_email = source;
+    }
+
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+
+        const fieldType = fields.getFieldType(fld.type);
+        if (fieldType.grouped) {
+            for (const optionKey in fld.groupedOptions) {
+                const option = fld.groupedOptions[optionKey];
+
+                if (option.column in subscription) {
+                    subscription['source_' + option.column] = source;
+                }
+            }
+        } else {
+            if (fldCol in subscription) {
+                subscription['source_' + fldCol] = source;
+            }
+        }
+    }
+}
+
+async function _update(tx, listId, existing, filteredEntity) {
+    if ('status' in filteredEntity) {
+        if (existing.status !== filteredEntity.status) {
+            filteredEntity.status_change = new Date();
+        }
+    }
+
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
+
+    if ('status' in filteredEntity) {
+        let countIncrement = 0;
+        if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
+            countIncrement = -1;
+        } else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
+            countIncrement = 1;
+        }
+
+        if (countIncrement) {
+            await tx('lists').where('id', listId).increment('subscribers', countIncrement);
+        }
+    }
+}
+
+async function _create(tx, listId, filteredEntity) {
+    const ids = await tx(getSubscriptionTableName(listId)).insert(filteredEntity);
+    const id = ids[0];
+
+    if (filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).increment('subscribers', 1);
+    }
+
+    return id;
+}
+
+/*
+    Adds a new subscription. Returns error if a subscription with the same email address is already present and is not unsubscribed.
+    If it is unsubscribed and meta.updateOfUnsubscribedAllowed, the existing subscription is changed based on the provided data.
+    If meta.updateAllowed is true, it updates even an active subscription.
+ */
+async function createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+    const allowedKeys = getAllowedKeys(groupedFieldsMap);
+
+    await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, true);
+
+    const filteredEntity = filterObject(entity, allowedKeys);
+    filteredEntity.status_change = new Date();
+
+    ungroupSubscription(groupedFieldsMap, filteredEntity);
+
+    updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
+
+    filteredEntity.opt_in_ip = meta && meta.ip;
+    filteredEntity.opt_in_country = meta && meta.country;
+
+    if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
+        await _update(tx, listId, meta.existing, filteredEntity);
+        meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
+        return meta.existing.id;
+    } else {
+        filteredEntity.cid = shortid.generate();
+
+        if (meta) {
+            meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
+        }
+
+        return await _create(tx, listId, filteredEntity);
+    }
+}
+
+async function create(context, listId, entity, source, meta) {
+    return await knex.transaction(async tx => {
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        return await createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta);
+    });
+}
+
+async function updateWithConsistencyCheck(context, listId, entity, source) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        const allowedKeys = getAllowedKeys(groupedFieldsMap);
+
+        groupSubscription(groupedFieldsMap, existing);
+
+        const existingHash = hashByAllowedKeys(allowedKeys, existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, null, false);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+
+        ungroupSubscription(groupedFieldsMap, filteredEntity);
+
+        updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
+
+        await _update(tx, listId, existing, filteredEntity);
+    });
+}
+
+async function _removeAndGetTx(tx, context, listId, existing) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).del();
+
+    if (existing.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).decrement('subscribers', 1);
+    }
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', id).first();
+        await _removeAndGetTx(tx, context, listId, existing);
+    });
+}
+
+async function removeByEmailAndGet(context, listId, email) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
+        return await _removeAndGetTx(tx, context, listId, existing);
+    });
+}
+
+
+async function _changeStatusTx(tx, context, listId, existing, newStatus) {
+    enforce(newStatus !== SubscriptionStatus.SUBSCRIBED);
+
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update({
+        status: newStatus
+    });
+
+    if (existing.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).decrement('subscribers', 1);
+    }
+}
+
+async function _unsubscribeExistingAndGetTx(tx, context, listId, existing) {
+    if (!(existing && existing.status === SubscriptionStatus.SUBSCRIBED)) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    await _changeStatusTx(tx, context, listId, existing, SubscriptionStatus.UNSUBSCRIBED);
+
+    existing.status = SubscriptionStatus.SUBSCRIBED;
+
+    return existing;
+}
+
+async function unsubscribeByIdAndGet(context, listId, subscriptionId) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+        return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
+    });
+}
+
+async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaignCid) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('cid', subscriptionCid).first();
+
+        if (campaignCid) {
+            await campaigns.changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaignCid, listId, existing.id, SubscriptionStatus.UNSUBSCRIBED);
+        }
+
+        return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
+    });
+}
+
+async function unsubscribeByEmailAndGetTx(tx, context, listId, email) {
+    const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
+    return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
+}
+
+async function unsubscribeByEmailAndGet(context, listId, email) {
+    return await knex.transaction(async tx => {
+        return await unsubscribeByEmailAndGetTx(tx, context, listId, email);
+    });
+}
+
+async function changeStatusTx(tx, context, listId, subscriptionId, subscriptionStatus) {
+    const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+    await _changeStatusTx(tx, context, listId, existing, subscriptionStatus);
+}
+
+
+
+async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (existing.email !== emailNew) {
+            await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(emailNew)).del();
+
+            await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
+                email: emailNew
+            });
+
+            existing.email = emailNew;
+        }
+
+        return existing;
+    });
+}
+
+async function updateManaged(context, listId, cid, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+        const update = {};
+        for (const key in groupedFieldsMap) {
+            const fld = groupedFieldsMap[key];
+
+            if (fld.order_manage) {
+                update[key] = entity[key];
+            }
+
+            fieldTypes[fld.type].afterJSON(fld, update);
+        }
+
+        ungroupSubscription(groupedFieldsMap, update);
+
+        await tx(getSubscriptionTableName(listId)).where('cid', cid).update(update);
+    });
+}
+
+
+async function getListsWithEmail(context, email) {
+    // FIXME - this methods is rather suboptimal if there are many lists. It quite needs permission caching in shares.js
+
+    return await knex.transaction(async tx => {
+        const lsts = await tx('lists').select(['id', 'name']);
+        const result = [];
+
+        for (const list of lsts) {
+            await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
+            const entity = await tx(getSubscriptionTableName(list.id)).where('email', email).first();
+            if (entity) {
+                result.push(list);
+            }
+        }
+
+        return result;
+    });
+}
+
+module.exports.getSubscriptionTableName = getSubscriptionTableName;
+module.exports.hashByList = hashByList;
+module.exports.getById = getById;
+module.exports.getByCidTx = getByCidTx;
+module.exports.getByCid = getByCid;
+module.exports.getByEmail = getByEmail;
+module.exports.list = list;
+module.exports.listIterator = listIterator;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listTestUsersDTAjax = listTestUsersDTAjax;
+module.exports.serverValidate = serverValidate;
+module.exports.create = create;
+module.exports.getGroupedFieldsMapTx = getGroupedFieldsMapTx;
+module.exports.createTxWithGroupedFieldsMap = createTxWithGroupedFieldsMap;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeByEmailAndGet = removeByEmailAndGet;
+module.exports.unsubscribeByCidAndGet = unsubscribeByCidAndGet;
+module.exports.unsubscribeByIdAndGet = unsubscribeByIdAndGet;
+module.exports.unsubscribeByEmailAndGet = unsubscribeByEmailAndGet;
+module.exports.unsubscribeByEmailAndGetTx = unsubscribeByEmailAndGetTx;
+module.exports.updateAddressAndGet = updateAddressAndGet;
+module.exports.updateManaged = updateManaged;
+module.exports.getListsWithEmail = getListsWithEmail;
+module.exports.changeStatusTx = changeStatusTx;

server/models/templates.js

@@ -0,0 +1,123 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const reports = require('./reports');
+const files = require('./files');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'type', 'data', 'html', 'text', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getByIdTx(tx, context, id, withPermissions = true) {
+    await shares.enforceEntityPermissionTx(tx, context, 'template', id, 'view');
+    const entity = await tx('templates').where('id', id).first();
+    entity.data = JSON.parse(entity.data);
+
+    if (withPermissions) {
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'template', id);
+    }
+
+    return entity;
+}
+
+async function getById(context, id, withPermissions = true) {
+    return await knex.transaction(async tx => {
+        return await getByIdTx(tx, context, id, withPermissions);
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'template', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('templates').innerJoin('namespaces', 'namespaces.id', 'templates.namespace'),
+        [ 'templates.id', 'templates.name', 'templates.description', 'templates.type', 'templates.created', 'namespaces.name' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, entity) {
+    await namespaceHelpers.validateEntity(tx, entity);
+
+    // We don't check contents of the "data" because it is processed solely on the client. The client generates the HTML code we use when sending out campaigns.
+
+    entity.data = JSON.stringify(entity.data);
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createTemplate');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const ids = await tx('templates').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'template', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'template', entity.id, 'edit');
+
+        const existing = await tx('templates').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.data = JSON.parse(existing.data);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'template', 'createTemplate', 'delete');
+
+        await tx('templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'template', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'template', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            {
+                entityTypeId: 'campaign',
+                query: tx => tx('template_dep_campaigns')
+                    .where('template_dep_campaigns.template', id)
+                    .innerJoin('campaigns', 'template_dep_campaigns.campaign', 'campaigns.id')
+                    .select(['campaigns.id', 'campaigns.name'])
+            }
+        ]);
+
+        await files.removeAllTx(tx, context, 'template', 'file', id);
+
+        await tx('templates').where('id', id).del();
+    });
+}
+
+module.exports.hash = hash;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/triggers.js

@@ -0,0 +1,145 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const {EntityVals, EventVals, Entity} = require('../../shared/triggers');
+const campaigns = require('./campaigns');
+
+const allowedKeys = new Set(['name', 'description', 'entity', 'event', 'seconds', 'enabled', 'source_campaign']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, campaignId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'viewTriggers');
+
+        const entity = await tx('triggers').where({campaign: campaignId, id}).first();
+
+        return entity;
+    });
+}
+
+async function listByCampaignDTAjax(context, campaignId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'viewTriggers');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('triggers')
+                .innerJoin('campaigns', 'campaigns.id', 'triggers.campaign')
+                .where('triggers.campaign', campaignId),
+            [ 'triggers.id', 'triggers.name', 'triggers.description', 'triggers.entity', 'triggers.event', 'triggers.seconds', 'triggers.enabled' ]
+        );
+    });
+}
+
+async function listByListDTAjax(context, listId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['viewTriggers'] }],
+        params,
+        builder => builder
+            .from('triggers')
+            .innerJoin('campaigns', 'campaigns.id', 'triggers.campaign')
+            .innerJoin('campaign_lists', 'campaign_lists.campaign', 'campaigns.id')
+            .where('campaign_lists.list', listId),
+        [ 'triggers.id', 'triggers.name', 'triggers.description', 'campaigns.name', 'triggers.entity', 'triggers.event', 'triggers.seconds', 'triggers.enabled', 'triggers.campaign' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, context, campaignId, entity) {
+    enforce(Number.isInteger(entity.seconds));
+    enforce(entity.seconds >= 0, 'Seconds must not be negative');
+    enforce(entity.entity in EntityVals, 'Invalid entity');
+    enforce(entity.event in EventVals[entity.entity], 'Invalid event');
+
+    if (entity.entity === Entity.CAMPAIGN) {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.source_campaign, 'view');
+    }
+
+    await campaigns.enforceSendPermissionTx(tx, context, campaignId);
+}
+
+async function create(context, campaignId, entity) {
+    return await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'manageTriggers');
+
+        await _validateAndPreprocess(tx, context, campaignId, entity);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.campaign = campaignId;
+
+        const ids = await tx('triggers').insert(filteredEntity);
+        const id = ids[0];
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, campaignId, entity) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'manageTriggers');
+
+        const existing = await tx('triggers').where({campaign: campaignId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, context, campaignId, entity);
+
+        await tx('triggers').where({campaign: campaignId, id: entity.id}).update(filterObject(entity, allowedKeys));
+    });
+}
+
+async function removeTx(tx, context, campaignId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'manageTriggers');
+
+    const existing = await tx('triggers').where({campaign: campaignId, id}).first();
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    await tx('trigger_messages').where({trigger: id}).del();
+    await tx('triggers').where('id', id).del();
+}
+
+async function remove(context, campaignId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, campaignId, id);
+    });
+}
+
+async function removeAllByCampaignIdTx(tx, context, campaignId) {
+    const entities = await tx('triggers').where('campaign', campaignId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, campaignId, entity.id);
+    }
+}
+
+
+
+// This is to handle circular dependency with campaigns.js
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listByCampaignDTAjax = listByCampaignDTAjax;
+module.exports.listByListDTAjax = listByListDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.removeTx = removeTx;
+module.exports.remove = remove;
+module.exports.removeAllByCampaignIdTx = removeAllByCampaignIdTx;

server/models/users.js

@@ -0,0 +1,439 @@
+'use strict';
+
+const config = require('config');
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const passwordValidator = require('../../shared/password-validator')();
+const dtHelpers = require('../lib/dt-helpers');
+const tools = require('../lib/tools');
+const crypto = require('crypto');
+const settings = require('./settings');
+const {getTrustedUrl} = require('../lib/urls');
+const { tUI } = require('../lib/translate');
+
+const bluebird = require('bluebird');
+
+const bcrypt = require('bcrypt-nodejs');
+const bcryptHash = bluebird.promisify(bcrypt.hash);
+const bcryptCompare = bluebird.promisify(bcrypt.compare);
+
+const mailers = require('../lib/mailers');
+
+const passport = require('../lib/passport');
+
+const namespaceHelpers = require('../lib/namespace-helpers');
+
+const allowedKeys = new Set(['username', 'name', 'email', 'password', 'namespace', 'role']);
+const ownAccountAllowedKeys = new Set(['name', 'email', 'password']);
+const allowedKeysExternal = new Set(['username', 'namespace', 'role']);
+const hashKeys = new Set(['username', 'name', 'email', 'namespace', 'role']);
+const shares = require('./shares');
+const contextHelpers = require('../lib/context-helpers');
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function _getBy(context, key, value, extraColumns = []) {
+    const columns = ['id', 'username', 'name', 'email', 'namespace', 'role', ...extraColumns];
+
+    const user = await knex('users').select(columns).where(key, value).first();
+
+    if (!user) {
+        shares.throwPermissionDenied();
+    }
+
+    await shares.enforceEntityPermission(context, 'namespace', user.namespace, 'manageUsers');
+
+    return user;
+}
+
+async function getById(context, id) {
+    return await _getBy(context, 'id', id);
+}
+
+async function serverValidate(context, data, isOwnAccount) {
+    const result = {};
+
+    if (!isOwnAccount) {
+        await shares.enforceTypePermission(context, 'namespace', 'manageUsers');
+    }
+
+    if (!isOwnAccount && data.username) {
+        const query = knex('users').select(['id']).where('username', data.username);
+
+        if (data.id) {
+            // Id is not set in entity creation form
+            query.andWhereNot('id', data.id);
+        }
+
+        const user = await query.first();
+        result.username = {
+            exists: !!user
+        };
+    }
+
+    if (isOwnAccount && data.currentPassword) {
+        const user = await knex('users').select(['id', 'password']).where('id', data.id).first();
+
+        result.currentPassword = {};
+        result.currentPassword.incorrect = !await bcryptCompare(data.currentPassword, user.password);
+    }
+
+    if (data.email) {
+        const query = knex('users').select(['id']).where('email', data.email);
+
+        if (data.id) {
+            // Id is not set in entity creation form
+            query.andWhereNot('id', data.id);
+        }
+
+        const user = await query.first();
+
+        result.email = {};
+        result.email.invalid = await tools.validateEmail(data.email) !== 0;
+        result.email.exists = !!user;
+    }
+
+    return result;
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'namespace', requiredOperations: ['manageUsers'] }],
+        params,
+        builder => builder
+            .from('users')
+            .innerJoin('namespaces', 'namespaces.id', 'users.namespace')
+            .innerJoin('generated_role_names', 'generated_role_names.role', 'users.role')
+            .where('generated_role_names.entity_type', 'global'),
+        [ 'users.id', 'users.username', 'users.name', 'namespaces.name', 'generated_role_names.name' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, entity, isCreate, isOwnAccount) {
+    enforce(await tools.validateEmail(entity.email) === 0, 'Invalid email');
+
+    await namespaceHelpers.validateEntity(tx, entity);
+
+    const otherUserWithSameEmailQuery = tx('users').where('email', entity.email);
+    if (entity.id) {
+        otherUserWithSameEmailQuery.andWhereNot('id', entity.id);
+    }
+
+    if (await otherUserWithSameEmailQuery.first()) {
+        throw new interoperableErrors.DuplicitEmailError();
+    }
+
+
+    if (!isOwnAccount) {
+        const otherUserWithSameUsernameQuery = tx('users').where('username', entity.username);
+        if (entity.id) {
+            otherUserWithSameUsernameQuery.andWhereNot('id', entity.id);
+        }
+
+        if (await otherUserWithSameUsernameQuery.first()) {
+            throw new interoperableErrors.DuplicitNameError();
+        }
+    }
+
+    enforce(entity.role in config.roles.global, 'Unknown role');
+
+    enforce(!isCreate || entity.password.length > 0, 'Password not set');
+
+    if (entity.password) {
+        const passwordValidatorResults = passwordValidator.test(entity.password);
+        if (passwordValidatorResults.errors.length > 0) {
+            // This is not an interoperable error because this is not supposed to happen unless the client is tampered with.
+            throw new Error('Invalid password');
+        }
+
+        entity.password = await bcryptHash(entity.password, null, null);
+    } else {
+        delete entity.password;
+    }
+}
+
+async function create(context, user) {
+    let id;
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', user.namespace, 'manageUsers');
+
+        if (passport.isAuthMethodLocal) {
+            await _validateAndPreprocess(tx, user, true);
+
+            const ids = await tx('users').insert(filterObject(user, allowedKeys));
+            id = ids[0];
+
+        } else {
+            const filteredUser = filterObject(user, allowedKeysExternal);
+            enforce(user.role in config.roles.global, 'Unknown role');
+
+            await namespaceHelpers.validateEntity(tx, user);
+
+            const ids = await tx('users').insert(filteredUser);
+            id = ids[0];
+        }
+
+        await shares.rebuildPermissionsTx(tx, { userId: id });
+    });
+
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, user, isOwnAccount) {
+    await knex.transaction(async tx => {
+        const existing = await tx('users').where('id', user.id).first();
+        if (!existing) {
+            shares.throwPermissionDenied();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== user.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        if (!isOwnAccount) {
+            await shares.enforceEntityPermissionTx(tx, context, 'namespace', user.namespace, 'manageUsers');
+            await shares.enforceEntityPermissionTx(tx, context, 'namespace', existing.namespace, 'manageUsers');
+        }
+
+        if (passport.isAuthMethodLocal) {
+            await _validateAndPreprocess(tx, user, false, isOwnAccount);
+
+            if (isOwnAccount && user.password) {
+                if (!await bcryptCompare(user.currentPassword, existing.password)) {
+                    throw new interoperableErrors.IncorrectPasswordError();
+                }
+            }
+
+            await tx('users').where('id', user.id).update(filterObject(user, isOwnAccount ? ownAccountAllowedKeys : allowedKeys));
+        } else {
+            enforce(!isOwnAccount, 'Local user management is required');
+            enforce(user.role in config.roles.global, 'Unknown role');
+            await namespaceHelpers.validateEntity(tx, user);
+
+            await tx('users').where('id', user.id).update(filterObject(user, allowedKeysExternal));
+        }
+
+        // Removes the default shares based on the user role and rebuilds permissions.
+        // rebuildPermissions adds the default shares based on the user role, which will reflect the changes
+        // done to the user.
+        if (existing.namespace !== user.namespace || existing.role !== user.role) {
+            await shares.removeDefaultShares(tx, existing);
+        }
+
+        await shares.rebuildPermissionsTx(tx, { userId: user.id });
+    });
+}
+
+async function remove(context, userId) {
+    enforce(userId !== 1, 'Admin cannot be deleted');
+    enforce(context.user.id !== userId, 'User cannot delete himself/herself');
+
+    await knex.transaction(async tx => {
+        const existing = await tx('users').where('id', userId).first();
+        if (!existing) {
+            shares.throwPermissionDenied();
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', existing.namespace, 'manageUsers');
+
+        await tx('users').where('id', userId).del();
+    });
+}
+
+async function getByAccessToken(accessToken) {
+    return await _getBy(contextHelpers.getAdminContext(), 'access_token', accessToken);
+}
+
+async function getByUsername(username) {
+    return await _getBy(contextHelpers.getAdminContext(), 'username', username);
+}
+
+async function getByUsernameIfPasswordMatch(username, password) {
+    try {
+        const user = await _getBy(contextHelpers.getAdminContext(), 'username', username, ['password']);
+
+        if (!await bcryptCompare(password, user.password)) {
+            throw new interoperableErrors.IncorrectPasswordError();
+        }
+
+        delete user.password;
+
+        return user;
+
+    } catch (err) {
+        if (err instanceof interoperableErrors.NotFoundError) {
+            throw new interoperableErrors.IncorrectPasswordError();
+        }
+
+        throw err;
+    }
+}
+
+async function getAccessToken(userId) {
+    const user = await _getBy(contextHelpers.getAdminContext(), 'id', userId, ['access_token']);
+    return user.access_token;
+}
+
+async function resetAccessToken(userId) {
+    const token = crypto.randomBytes(20).toString('hex').toLowerCase();
+    await knex('users').where({id: userId}).update({access_token: token});
+    return token;
+}
+
+async function sendPasswordReset(language, usernameOrEmail) {
+    enforce(passport.isAuthMethodLocal, 'Local user management is required');
+
+    await knex.transaction(async tx => {
+        const user = await tx('users').where('username', usernameOrEmail).orWhere('email', usernameOrEmail).select(['id', 'username', 'email', 'name']).first();
+
+        if (user) {
+            const resetToken = crypto.randomBytes(16).toString('base64').replace(/[^a-z0-9]/gi, '');
+
+            await tx('users').where('id', user.id).update({
+                reset_token: resetToken,
+                reset_expire: new Date(Date.now() + 60 * 60 * 1000)
+            });
+
+            const { adminEmail } = await settings.get(contextHelpers.getAdminContext(), ['adminEmail']);
+
+            const mailer = await mailers.getOrCreateMailer();
+            await mailer.sendTransactionalMail({
+                from: {
+                    address: adminEmail
+                },
+                to: {
+                    address: user.email
+                },
+                subject: tUI('account.passwordChangeRequest', language)
+            }, {
+                html: 'emails/password-reset-html.hbs',
+                text: 'emails/password-reset-text.hbs',
+                data: {
+                    title: 'Mailtrain',
+                    username: user.username,
+                    name: user.name,
+                    confirmUrl: getTrustedUrl(`/account/reset/${encodeURIComponent(user.username)}/${encodeURIComponent(resetToken)}`)
+                }
+            });
+        }
+        // We intentionally silently ignore the situation when user is not found. This is not to reveal if a user exists in the system.
+    });
+}
+
+async function isPasswordResetTokenValid(username, resetToken) {
+    enforce(passport.isAuthMethodLocal, 'Local user management is required');
+
+    const user = await knex('users').select(['id']).where({username, reset_token: resetToken}).andWhere('reset_expire', '>', new Date()).first();
+    return !!user;
+}
+
+async function resetPassword(username, resetToken, password) {
+    enforce(passport.isAuthMethodLocal, 'Local user management is required');
+
+    await knex.transaction(async tx => {
+        const user = await tx('users').select(['id']).where({
+            username,
+            reset_token: resetToken
+        }).andWhere('reset_expire', '>', new Date()).first();
+
+        if (user) {
+            const passwordValidatorResults = passwordValidator.test(password);
+            if (passwordValidatorResults.errors.length > 0) {
+                // This is not an interoperable error because this is not supposed to happen unless the client is tampered with.
+                throw new Error('Invalid password');
+            }
+
+            password = await bcryptHash(password, null, null);
+
+            await tx('users').where({username}).update({
+                password,
+                reset_token: null,
+                reset_expire: null
+            });
+        } else {
+            throw new interoperableErrors.InvalidTokenError();
+        }
+    });
+}
+
+
+
+const restrictedAccessTokenMethods = {};
+const restrictedAccessTokens = new Map();
+
+function registerRestrictedAccessTokenMethod(method, getHandlerFromParams) {
+    restrictedAccessTokenMethods[method] = getHandlerFromParams;
+}
+
+async function getRestrictedAccessToken(context, method, params) {
+    const token = crypto.randomBytes(24).toString('hex').toLowerCase();
+    const tokenEntry = {
+        token,
+        userId: context.user.id,
+        handler: await restrictedAccessTokenMethods[method](params),
+        expires: Date.now() + 120 * 1000
+    };
+
+    restrictedAccessTokens.set(token, tokenEntry);
+
+    return token;
+}
+
+async function refreshRestrictedAccessToken(context, token) {
+    const tokenEntry = restrictedAccessTokens.get(token);
+
+    if (tokenEntry && tokenEntry.userId === context.user.id) {
+        tokenEntry.expires = Date.now() + 120 * 1000
+    } else {
+        shares.throwPermissionDenied();
+    }
+}
+
+async function getByRestrictedAccessToken(token) {
+    const now = Date.now();
+    for (const entry of restrictedAccessTokens.values()) {
+        if (entry.expires < now) {
+            restrictedAccessTokens.delete(entry.token);
+        }
+    }
+
+    const tokenEntry = restrictedAccessTokens.get(token);
+
+    if (tokenEntry) {
+        const user = await getById(contextHelpers.getAdminContext(), tokenEntry.userId);
+        user.restrictedAccessHandler = tokenEntry.handler;
+        user.restrictedAccessToken = tokenEntry.token;
+
+        return user;
+
+    } else {
+        shares.throwPermissionDenied();
+    }
+}
+
+
+module.exports.listDTAjax = listDTAjax;
+module.exports.remove = remove;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.create = create;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.serverValidate = serverValidate;
+module.exports.getByAccessToken = getByAccessToken;
+module.exports.getByUsername = getByUsername;
+module.exports.getByUsernameIfPasswordMatch = getByUsernameIfPasswordMatch;
+module.exports.getAccessToken = getAccessToken;
+module.exports.resetAccessToken = resetAccessToken;
+module.exports.sendPasswordReset = sendPasswordReset;
+module.exports.isPasswordResetTokenValid = isPasswordResetTokenValid;
+module.exports.resetPassword = resetPassword;
+module.exports.getByRestrictedAccessToken = getByRestrictedAccessToken;
+module.exports.getRestrictedAccessToken = getRestrictedAccessToken;
+module.exports.refreshRestrictedAccessToken = refreshRestrictedAccessToken;
+module.exports.registerRestrictedAccessTokenMethod = registerRestrictedAccessTokenMethod;

server/package.json

@@ -0,0 +1,114 @@
+{
+  "name": "mailtrain",
+  "private": true,
+  "version": "2.0.0",
+  "description": "Self hosted email newsletter app - server",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js",
+    "sqlinit": "node setup/sql/init.js",
+    "sqldump": "node setup/sql/dump.js | sed -e '/^\\/\\*.*\\*\\/;$/d' -e 's/.[0-9]\\{4\\}-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]./NOW()/g' > setup/sql/mailtrain${DUMP_NAME_SUFFIX}.sql",
+    "sqldrop": "node setup/sql/drop.js",
+    "sqlgen": "npm run sqldrop && DB_FROM_START=Y npm run sqlinit && npm run sqldump",
+    "sqldumptest": "NODE_ENV=test DUMP_NAME_SUFFIX=-test npm run sqldump",
+    "sqlresettest": "NODE_ENV=test npm run sqldrop && NODE_ENV=test npm run sqlinit",
+    "starttest": "NODE_ENV=test node index.js",
+    "_e2e": "NODE_ENV=test node test/e2e/index.js",
+    "e2e": "npm run sqlresettest && npm run _e2e"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/Mailtrain-org/mailtrain.git"
+  },
+  "license": "GPL-3.0",
+  "homepage": "https://mailtrain.org/",
+  "engines": {
+    "node": ">=10.0.0"
+  },
+  "devDependencies": {
+    "babel-eslint": "^10.0.1",
+    "chai": "^4.2.0",
+    "eslint-config-nodemailer": "^1.2.0",
+    "grunt": "^1.0.3",
+    "grunt-cli": "^1.3.2",
+    "grunt-eslint": "^21.0.0",
+    "lodash": "^4.17.11",
+    "mocha": "^5.2.0",
+    "phantomjs-prebuilt": "^2.1.16",
+    "selenium-webdriver": "^3.5.0",
+    "url-pattern": "^1.0.3"
+  },
+  "optionalDependencies": {
+    "posix": "^4.1.2"
+  },
+  "dependencies": {
+    "aws-sdk": "^2.358.0",
+    "bcrypt-nodejs": "0.0.3",
+    "bluebird": "^3.5.3",
+    "body-parser": "^1.18.3",
+    "bounce-handler": "^7.3.2-fork.2",
+    "compression": "^1.7.3",
+    "config": "^2.0.1",
+    "connect-flash": "^0.1.1",
+    "cookie-parser": "^1.4.3",
+    "cors": "^2.8.5",
+    "crypto": "^1.0.1",
+    "csurf": "^1.9.0",
+    "csv-parse": "^3.2.0",
+    "csv-stringify": "^4.3.1",
+    "device": "^0.3.9",
+    "dompurify": "^1.0.8",
+    "escape-html": "^1.0.3",
+    "escape-string-regexp": "^1.0.5",
+    "express": "^4.16.4",
+    "express-locale": "^1.0.5",
+    "express-session": "^1.15.6",
+    "faker": "^4.1.0",
+    "feedparser-promised": "^2.0.0",
+    "fs-extra": "^7.0.1",
+    "fs-extra-promise": "^1.0.1",
+    "geoip-ultralight": "^0.1.5",
+    "gm": "^1.23.1",
+    "handlebars": "^4.0.12",
+    "hbs": "^4.0.1",
+    "he": "^1.2.0",
+    "html-to-text": "^4.0.0",
+    "humanize": "0.0.9",
+    "i18next": "^12.0.0",
+    "i18next-node-fs-backend": "^2.1.0",
+    "isemail": "^3.2.0",
+    "jsdom": "^13.0.0",
+    "juice": "^5.0.1",
+    "knex": "^0.15.2",
+    "libmime": "^4.0.1",
+    "mailparser": "^2.4.3",
+    "memory-cache": "^0.2.0",
+    "mjml": "^4.2.0",
+    "moment": "^2.22.2",
+    "moment-timezone": "^0.5.23",
+    "morgan": "^1.9.1",
+    "multer": "^1.4.1",
+    "mysql2": "^1.6.4",
+    "node-ipc": "^9.1.1",
+    "node-mocks-http": "^1.7.3",
+    "node-object-hash": "^1.4.1",
+    "nodeify": "^1.0.1",
+    "nodemailer": "^4.6.8",
+    "nodemailer-openpgp": "^1.2.0",
+    "npmlog": "^4.1.2",
+    "nyc": "^13.1.0",
+    "openpgp": "^4.2.1",
+    "passport": "^0.4.0",
+    "passport-local": "^1.0.0",
+    "premailer-api": "^1.0.4",
+    "request": "^2.88.0",
+    "request-promise": "^4.2.2",
+    "serve-favicon": "^2.5.0",
+    "shortid": "^2.2.14",
+    "slugify": "^1.3.3",
+    "smtp-server": "^3.4.7",
+    "toml": "^2.3.3",
+    "try-require": "^1.2.1",
+    "xmldom": "^0.1.27"
+  }
+}

server/protected/reports/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.md

server/protected/reports/README.md

@@ -0,0 +1 @@
+This directory serves for generated reports.

server/routes/api.js

@@ -0,0 +1,283 @@
+'use strict';
+
+const config = require('config');
+const lists = require('../models/lists');
+const tools = require('../lib/tools');
+const blacklist = require('../models/blacklist');
+const fields = require('../models/fields');
+const { SubscriptionStatus, SubscriptionSource } = require('../../shared/lists');
+const subscriptions = require('../models/subscriptions');
+const confirmations = require('../models/confirmations');
+const log = require('../lib/log');
+const router = require('../lib/router-async').create();
+const mailHelpers = require('../lib/subscription-mail-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const contextHelpers = require('../lib/context-helpers');
+const shares = require('../models/shares');
+const slugify = require('slugify');
+const passport = require('../lib/passport');
+
+class APIError extends Error {
+    constructor(msg, status) {
+        super(msg);
+        this.status = status;
+    }
+}
+
+
+router.postAsync('/subscribe/:listCid', passport.loggedIn, async (req, res) => {
+    const list = await lists.getByCid(req.context, req.params.listCid);
+    await shares.enforceEntityPermission(req.context, 'list', list.id, 'manageSubscriptions');
+
+    const input = {};
+    Object.keys(req.body).forEach(key => {
+        input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
+    });
+
+    if (!input.EMAIL) {
+        throw new APIError('Missing EMAIL', 400);
+    }
+
+    const emailErr = await tools.validateEmail(input.EMAIL);
+    if (emailErr) {
+        const errMsg = tools.validateEmailGetMessage(emailErr, input.email);
+        log.error('API', errMsg);
+        throw new APIError(errMsg, 400);
+    }
+
+    const subscription = await fields.fromAPI(req.context, list.id, input);
+
+    if (input.TIMEZONE) {
+        subscription.tz = (input.TIMEZONE || '').toString().trim();
+    }
+
+    if (/^(yes|true|1)$/i.test(input.FORCE_SUBSCRIBE)) {
+        subscription.status = SubscriptionStatus.SUBSCRIBED;
+    }
+
+    if (/^(yes|true|1)$/i.test(input.REQUIRE_CONFIRMATION)) { // if REQUIRE_CONFIRMATION is set, we assume that the user is not subscribed and will be subscribed
+        const data = {
+            email: input.EMAIL,
+            subscriptionData: subscription
+        };
+
+        const confirmCid = await confirmations.addConfirmation(list.id, 'subscribe', req.ip, data);
+        await mailHelpers.sendConfirmSubscription(config.language, list, input.EMAIL, confirmCid, subscription);
+
+        res.status(200);
+        res.json({
+            data: {
+                id: confirmCid
+            }
+        });
+    } else {
+        subscription.email = input.EMAIL;
+
+        const meta = {
+            updateAllowed: true,
+            subscribeIfNoExisting: true
+        };
+
+        await subscriptions.create(req.context, list.id, subscription, SubscriptionSource.API, meta);
+
+        res.status(200);
+        res.json({
+            data: {
+                id: meta.cid
+            }
+        });
+    }
+});
+
+
+router.postAsync('/unsubscribe/:listCid', passport.loggedIn, async (req, res) => {
+    const list = await lists.getByCid(req.context, req.params.listCid);
+    const input = {};
+    Object.keys(req.body).forEach(key => {
+        input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
+    });
+
+    if (!input.EMAIL) {
+        throw new APIError('Missing EMAIL', 400);
+    }
+
+    const subscription = await subscriptions.unsubscribeByEmailAndGet(req.context, list.id, input.EMAIL);
+
+    res.status(200);
+    res.json({
+        data: {
+            id: subscription.id,
+            unsubscribed: true
+        }
+    });
+});
+
+
+router.postAsync('/delete/:listCid', passport.loggedIn, async (req, res) => {
+    const list = await lists.getByCid(req.context, req.params.listCid);
+    const input = {};
+    Object.keys(req.body).forEach(key => {
+        input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
+    });
+
+    if (!input.EMAIL) {
+        throw new APIError('Missing EMAIL', 400);
+    }
+
+    const subscription = await subscriptions.removeByEmailAndGet(req.context, list.id, input.EMAIL);
+
+    res.status(200);
+    res.json({
+        data: {
+            id: subscription.id,
+            deleted: true
+        }
+    });
+});
+
+
+router.getAsync('/subscriptions/:listCid', passport.loggedIn, async (req, res) => {
+    const list = await lists.getByCid(req.context, req.params.listCid);
+    const start = parseInt(req.query.start || 0, 10);
+    const limit = parseInt(req.query.limit || 10000, 10);
+
+    const { subscriptions, total } = await subscriptions.list(list.id, false, start, limit);
+
+    res.status(200);
+    res.json({
+        data: {
+            total: total,
+            start: start,
+            limit: limit,
+            subscriptions
+        }
+    });
+});
+
+router.getAsync('/lists/:email', passport.loggedIn, async (req, res) => {
+    const lists = await subscriptions.getListsWithEmail(req.context, req.params.email);
+
+    res.status(200);
+    res.json({
+        data: lists
+    });
+});
+
+
+router.postAsync('/field/:listCid', passport.loggedIn, async (req, res) => {
+    const list = await lists.getByCid(req.context, req.params.listCid);
+    const input = {};
+    Object.keys(req.body).forEach(key => {
+        input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
+    });
+
+    const key = (input.NAME || '').toString().trim() || slugify('merge ' + name, '_').toUpperCase();
+    const visible = ['false', 'no', '0', ''].indexOf((input.VISIBLE || '').toString().toLowerCase().trim()) < 0;
+
+    const groupTemplate = (input.GROUP_TEMPLATE || '').toString().toLowerCase().trim();
+
+    let type = (input.TYPE || '').toString().toLowerCase().trim();
+    const settings = {};
+
+    if (type === 'checkbox') {
+        type = 'checkbox-grouped';
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'dropdown') {
+        type = 'dropdown-grouped';
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'radio') {
+        type = 'radio-grouped';
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'json') {
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'date-us') {
+        type = 'date';
+        settings.dateFormat = 'us';
+    } else if (type === 'date-eur') {
+        type = 'date';
+        settings.dateFormat = 'eur';
+    } else if (type === 'birthday-us') {
+        type = 'birthday';
+        settings.birthdayFormat = 'us';
+    } else if (type === 'birthday-eur') {
+        type = 'birthday';
+        settings.birthdayFormat = 'eur';
+    }
+
+    const field = {
+        name: (input.NAME || '').toString().trim(),
+        key,
+        default_value: (input.DEFAULT || '').toString().trim() || null,
+        type,
+        settings,
+        group: Number(input.GROUP) || null,
+        orderListBefore: visible ? 'end' : 'none',
+        orderSubscribeBefore: visible ? 'end' : 'none',
+        orderManageBefore: visible ? 'end' : 'none'
+    };
+
+    const id = await fields.create(req.context, list.id, field);
+
+    res.status(200);
+    res.json({
+        data: {
+            id,
+            tag: key
+        }
+    });
+});
+
+
+router.postAsync('/blacklist/add', passport.loggedIn, async (req, res) => {
+    let input = {};
+    Object.keys(req.body).forEach(key => {
+        input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
+    });
+    if (!(input.EMAIL) || (input.EMAIL === ''))  {
+        throw new Error('EMAIL argument is required');
+    }
+
+    await blacklist.add(req.context, input.EMAIL);
+
+    res.json({
+        data: []
+    });
+});
+
+
+router.postAsync('/blacklist/delete', passport.loggedIn, async (req, res) => {
+    let input = {};
+    Object.keys(req.body).forEach(key => {
+        input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
+    });
+    if (!(input.EMAIL) || (input.EMAIL === '')) {
+        throw new Error('EMAIL argument is required');
+    }
+
+    await blacklist.remove(req.oontext, input.EMAIL);
+
+    res.json({
+        data: []
+    });
+});
+
+
+router.getAsync('/blacklist/get', passport.loggedIn, async (req, res) => {
+    let start = parseInt(req.query.start || 0, 10);
+    let limit = parseInt(req.query.limit || 10000, 10);
+    let search = req.query.search || '';
+
+    const { emails, total } = await blacklist.search(req.context, start, limit, search);
+
+    return res.json({
+        data: {
+            total,
+            start: start,
+            limit: limit,
+            emails
+        }
+    });
+});
+
+
+module.exports = router;

server/routes/archive.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const router = require('../lib/router-async').create();
+const CampaignSender = require('../lib/campaign-sender');
+
+
+router.get('/:campaign/:list/:subscription', (req, res, next) => {
+    const cs = new CampaignSender();
+    cs.init({campaignCid: req.params.campaign})
+        .then(() => cs.getMessage(req.params.list, req.params.subscription))
+        .then(result => {
+            const {html} = result;
+
+            if (html.match(/<\/body\b/i)) {
+                res.render('partials/tracking-scripts', {
+                    layout: 'archive/layout-raw'
+                }, (err, scripts) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    const htmlWithScripts = scripts ? html.replace(/<\/body\b/i, match => scripts + match) : html;
+
+                    res.render('archive/view', {
+                        layout: 'archive/layout-raw',
+                        message: htmlWithScripts
+                    });
+                });
+
+            } else {
+                res.render('archive/view', {
+                    layout: 'archive/layout-wrapped',
+                    message: html
+                });
+            }
+
+        })
+        .catch(err => next(err));
+});
+
+module.exports = router;

server/routes/files.js

@@ -0,0 +1,13 @@
+'use strict';
+
+const router = require('../lib/router-async').create();
+const files = require('../models/files');
+const contextHelpers = require('../lib/context-helpers');
+
+router.getAsync('/:type/:subType/:entityId/:fileName', async (req, res) => {
+    const file = await files.getFileByFilename(contextHelpers.getAdminContext(), req.params.type, req.params.subType, req.params.entityId, req.params.fileName);
+    res.type(file.mimetype);
+    return res.download(file.path, file.name);
+});
+
+module.exports = router;

server/routes/index.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const passport = require('../lib/passport');
+const _ = require('../lib/translate')._;
+const clientHelpers = require('../lib/client-helpers');
+const { getTrustedUrl } = require('../lib/urls');
+const { AppType } = require('../../shared/app');
+
+const routerFactory = require('../lib/router-async');
+
+function getRouter(appType) {
+    const router = routerFactory.create();
+
+    if (appType === AppType.TRUSTED) {
+        router.getAsync('/*', passport.csrfProtection, async (req, res) => {
+            const mailtrainConfig = await clientHelpers.getAnonymousConfig(req.context, appType);
+            if (req.user) {
+                Object.assign(mailtrainConfig, await clientHelpers.getAuthenticatedConfig(req.context));
+            }
+
+            res.render('root', {
+                reactCsrfToken: req.csrfToken(),
+                mailtrainConfig: JSON.stringify(mailtrainConfig),
+                scriptFiles: [
+                    getTrustedUrl('mailtrain/root.js')
+                ]
+            });
+        });
+    }
+
+    return router;
+}
+
+
+module.exports.getRouter = getRouter;

server/routes/links.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const log = require('../lib/log');
+const config = require('config');
+const router = require('../lib/router-async').create();
+const links = require('../models/links');
+const interoperableErrors = require('../../shared/interoperable-errors');
+
+const trackImg = Buffer.from('R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', 'base64');
+
+router.getAsync('/:campaign/:list/:subscription', async (req, res) => {
+    res.writeHead(200, {
+        'Content-Type': 'image/gif',
+        'Content-Length': trackImg.length
+    });
+
+    res.end(trackImg);
+
+    await links.countLink(req.ip, req.headers['user-agent'], req.params.campaign, req.params.list, req.params.subscription, links.LinkId.OPEN);
+});
+
+
+router.getAsync('/:campaign/:list/:subscription/:link', async (req, res) => {
+    const link = await links.resolve(req.params.link);
+
+    if (link) {
+        // In Mailtrain v1 we would do the URL expansion here based on merge tags. We don't do it here anymore. Instead, the URLs are expanded when message is sent out (in links.updateLinks)
+        res.redirect(url);
+
+        await links.countLink(req.ip, req.headers['user-agent'], req.params.campaign, req.params.list, req.params.subscription, link.id);
+    } else {
+        log.error('Redirect', 'Unresolved URL: <%s>', req.url);
+        throw new interoperableErrors.NotFoundError('Oops, we couldn\'t find a link for the URL you clicked');
+    }
+});
+
+module.exports = router;

server/routes/reports.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const passport = require('../lib/passport');
+const reports = require('../models/reports');
+const reportHelpers = require('../lib/report-helpers');
+const shares = require('../models/shares');
+const contextHelpers = require('../lib/context-helpers');
+
+const router = require('../lib/router-async').create();
+
+const fileSuffixes = {
+    'text/html': '.html',
+    'text/csv': '.csv'
+};
+
+router.getAsync('/:id/download', passport.loggedIn, async (req, res) => {
+    await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
+
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
+
+    if (report.state == reports.ReportState.FINISHED) {
+        const headers = {
+            'Content-Disposition': 'attachment;filename=' + reportHelpers.nameToFileName(report.name) + (fileSuffixes[report.mime_type] || ''),
+            'Content-Type': report.mime_type
+        };
+
+        res.sendFile(reportHelpers.getReportContentFile(report), {headers: headers});
+
+    } else {
+        return res.status(404).send('Report not found');
+    }
+});
+
+module.exports = router;

server/routes/rest/account.js

@@ -0,0 +1,73 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const _ = require('../../lib/translate')._;
+const users = require('../../models/users');
+const contextHelpers = require('../../lib/context-helpers');
+
+const router = require('../../lib/router-async').create();
+
+
+router.getAsync('/account', passport.loggedIn, async (req, res) => {
+    const user = await users.getById(contextHelpers.getAdminContext(), req.user.id);
+    user.hash = users.hash(user);
+    return res.json(user);
+});
+
+router.postAsync('/account', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const data = req.body;
+    data.id = req.user.id;
+
+    await users.updateWithConsistencyCheck(contextHelpers.getAdminContext(), req.body, true);
+    return res.json();
+});
+
+router.postAsync('/account-validate', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const data = req.body;
+    data.id = req.user.id;
+
+    return res.json(await users.serverValidate(contextHelpers.getAdminContext(), data, true));
+});
+
+router.getAsync('/access-token', passport.loggedIn, async (req, res) => {
+    const accessToken = await users.getAccessToken(req.user.id);
+    return res.json(accessToken);
+
+});
+
+router.postAsync('/access-token-reset', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const accessToken = await users.resetAccessToken(req.user.id);
+    return res.json(accessToken);
+});
+
+
+router.post('/login', passport.csrfProtection, passport.restLogin);
+router.post('/logout', passport.csrfProtection, passport.restLogout);
+
+router.postAsync('/password-reset-send', passport.csrfProtection, async (req, res) => {
+    await users.sendPasswordReset(req.language, req.body.usernameOrEmail);
+    return res.json();
+});
+
+router.postAsync('/password-reset-validate', passport.csrfProtection, async (req, res) => {
+    const isValid = await users.isPasswordResetTokenValid(req.body.username, req.body.resetToken);
+    return res.json(isValid);
+});
+
+router.postAsync('/password-reset', passport.csrfProtection, async (req, res) => {
+    await users.resetPassword(req.body.username, req.body.resetToken, req.body.password);
+    return res.json();
+});
+
+router.postAsync('/restricted-access-token', passport.loggedIn, async (req, res) => {
+    const restrictedAccessToken = await users.getRestrictedAccessToken(req.context, req.body.method, req.body.params);
+    return res.json(restrictedAccessToken);
+
+});
+
+router.putAsync('/restricted-access-token', passport.loggedIn, async (req, res) => {
+    await users.refreshRestrictedAccessToken(req.context, req.body.token);
+    return res.json();
+});
+
+module.exports = router;

server/routes/rest/blacklist.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const blacklist = require('../../models/blacklist');
+
+const router = require('../../lib/router-async').create();
+
+
+router.postAsync('/blacklist-table', passport.loggedIn, async (req, res) => {
+    return res.json(await blacklist.listDTAjax(req.context, req.body));
+});
+
+router.postAsync('/blacklist', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await blacklist.add(req.context, req.body.email));
+});
+
+router.deleteAsync('/blacklist/:email', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await blacklist.remove(req.context, req.params.email);
+    return res.json();
+});
+
+router.postAsync('/blacklist-validate', passport.loggedIn, async (req, res) => {
+    return res.json(await blacklist.serverValidate(req.context, req.body));
+});
+
+module.exports = router;

server/routes/rest/campaigns.js

@@ -0,0 +1,98 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const campaigns = require('../../models/campaigns');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/campaigns-table', passport.loggedIn, async (req, res) => {
+    return res.json(await campaigns.listDTAjax(req.context, req.body));
+});
+
+router.postAsync('/campaigns-with-content-table', passport.loggedIn, async (req, res) => {
+    return res.json(await campaigns.listWithContentDTAjax(req.context, req.body));
+});
+
+router.postAsync('/campaigns-others-by-list-table/:campaignId/:listIds', passport.loggedIn, async (req, res) => {
+    return res.json(await campaigns.listOthersWhoseListsAreIncludedDTAjax(req.context, castToInteger(req.params.campaignId), req.params.listIds.split(';').map(x => castToInteger(x)), req.body));
+});
+
+router.postAsync('/campaigns-children/:campaignId', passport.loggedIn, async (req, res) => {
+    return res.json(await campaigns.listChildrenDTAjax(req.context, castToInteger(req.params.campaignId), req.body));
+});
+
+router.postAsync('/campaigns-test-users-table/:campaignId', passport.loggedIn, async (req, res) => {
+    return res.json(await campaigns.listTestUsersDTAjax(req.context, castToInteger(req.params.campaignId), req.body));
+});
+
+router.getAsync('/campaigns-settings/:campaignId', passport.loggedIn, async (req, res) => {
+    const campaign = await campaigns.getById(req.context, castToInteger(req.params.campaignId), true, campaigns.Content.WITHOUT_SOURCE_CUSTOM);
+    campaign.hash = campaigns.hash(campaign, campaigns.Content.WITHOUT_SOURCE_CUSTOM);
+    return res.json(campaign);
+});
+
+router.getAsync('/campaigns-stats/:campaignId', passport.loggedIn, async (req, res) => {
+    const campaign = await campaigns.getById(req.context, castToInteger(req.params.campaignId), true, campaigns.Content.SETTINGS_WITH_STATS);
+    return res.json(campaign);
+});
+
+router.getAsync('/campaigns-content/:campaignId', passport.loggedIn, async (req, res) => {
+    const campaign = await campaigns.getById(req.context, castToInteger(req.params.campaignId), true, campaigns.Content.ONLY_SOURCE_CUSTOM);
+    campaign.hash = campaigns.hash(campaign, campaigns.Content.ONLY_SOURCE_CUSTOM);
+    return res.json(campaign);
+});
+
+router.postAsync('/campaigns', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.create(req.context, req.body));
+});
+
+router.putAsync('/campaigns-settings/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.campaignId);
+
+    await campaigns.updateWithConsistencyCheck(req.context, entity, campaigns.Content.WITHOUT_SOURCE_CUSTOM);
+    return res.json();
+});
+
+router.putAsync('/campaigns-content/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.campaignId);
+
+    await campaigns.updateWithConsistencyCheck(req.context, entity, campaigns.Content.ONLY_SOURCE_CUSTOM);
+    return res.json();
+});
+
+router.deleteAsync('/campaigns/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await campaigns.remove(req.context, castToInteger(req.params.campaignId));
+    return res.json();
+});
+
+router.postAsync('/campaign-start/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.start(req.context, castToInteger(req.params.campaignId), null));
+});
+
+router.postAsync('/campaign-start-at/:campaignId/:dateTime', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.start(req.context, castToInteger(req.params.campaignId), new Date(Number.parseInt(req.params.dateTime))));
+});
+
+
+router.postAsync('/campaign-stop/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.stop(req.context, castToInteger(req.params.campaignId)));
+});
+
+router.postAsync('/campaign-reset/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.reset(req.context, castToInteger(req.params.campaignId)));
+});
+
+router.postAsync('/campaign-enable/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.enable(req.context, castToInteger(req.params.campaignId), null));
+});
+
+router.postAsync('/campaign-disable/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.disable(req.context, castToInteger(req.params.campaignId), null));
+});
+
+
+module.exports = router;

server/routes/rest/editors.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+
+const bluebird = require('bluebird');
+const premailerApi = require('premailer-api');
+const premailerPrepareAsync = bluebird.promisify(premailerApi.prepare);
+
+const router = require('../../lib/router-async').create();
+
+router.postAsync('/html-to-text', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    if (!req.body.html) {
+        return res.json({text: ''}); // Premailer crashes very hard when html is empty
+    }
+
+    const email = await premailerPrepareAsync({
+        html: req.body.html,
+        fetchHTML: false
+    });
+
+    res.json({text: email.text.replace(/%5B/g, '[').replace(/%5D/g, ']')});
+
+});
+
+module.exports = router;

server/routes/rest/fields.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const fields = require('../../models/fields');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/fields-table/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await fields.listDTAjax(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.postAsync('/fields-grouped-table/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await fields.listGroupedDTAjax(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.getAsync('/fields/:listId/:fieldId', passport.loggedIn, async (req, res) => {
+    const entity = await fields.getById(req.context, castToInteger(req.params.listId), castToInteger(req.params.fieldId));
+    entity.hash = fields.hash(entity);
+    return res.json(entity);
+});
+
+router.getAsync('/fields/:listId', passport.loggedIn, async (req, res) => {
+    const rows = await fields.list(req.context, castToInteger(req.params.listId));
+    return res.json(rows);
+});
+
+router.getAsync('/fields-grouped/:listId', passport.loggedIn, async (req, res) => {
+    const rows = await fields.listGrouped(req.context, castToInteger(req.params.listId));
+    return res.json(rows);
+});
+
+router.postAsync('/fields/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await fields.create(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.putAsync('/fields/:listId/:fieldId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.fieldId);
+
+    await fields.updateWithConsistencyCheck(req.context, castToInteger(req.params.listId), entity);
+    return res.json();
+});
+
+router.deleteAsync('/fields/:listId/:fieldId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await fields.remove(req.context, castToInteger(req.params.listId), castToInteger(req.params.fieldId));
+    return res.json();
+});
+
+router.postAsync('/fields-validate/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await fields.serverValidate(req.context, castToInteger(req.params.listId), req.body));
+});
+
+
+module.exports = router;

server/routes/rest/files.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const files = require('../../models/files');
+const router = require('../../lib/router-async').create();
+const fileHelpers = require('../../lib/file-helpers');
+const {castToInteger} = require('../../lib/helpers');
+
+router.postAsync('/files-table/:type/:subType/:entityId', passport.loggedIn, async (req, res) => {
+    return res.json(await files.listDTAjax(req.context, req.params.type, req.params.subType, castToInteger(req.params.entityId), req.body));
+});
+
+router.getAsync('/files-list/:type/:subType/:entityId', passport.loggedIn, async (req, res) => {
+    return res.json(await files.list(req.context, req.params.type, req.params.subType, castToInteger(req.params.entityId)));
+});
+
+
+router.getAsync('/files/:type/:subType/:fileId', passport.loggedIn, async (req, res) => {
+    const file = await files.getFileById(req.context, req.params.type, req.params.subType, castToInteger(req.params.fileId));
+    res.type(file.mimetype);
+    return res.download(file.path, file.name);
+});
+
+router.deleteAsync('/files/:type/:subType/:fileId', passport.loggedIn, async (req, res) => {
+    await files.removeFile(req.context, req.params.type, req.params.subType, castToInteger(req.params.fileId));
+    return res.json();
+});
+
+fileHelpers.installUploadHandler(router, '/files/:type/:subType/:entityId');
+
+module.exports = router;

server/routes/rest/forms.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const forms = require('../../models/forms');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/forms-table', passport.loggedIn, async (req, res) => {
+    return res.json(await forms.listDTAjax(req.context, req.body));
+});
+
+router.getAsync('/forms/:formId', passport.loggedIn, async (req, res) => {
+    const entity = await forms.getById(req.context, castToInteger(req.params.formId));
+    entity.hash = forms.hash(entity);
+    return res.json(entity);
+});
+
+router.postAsync('/forms', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await forms.create(req.context, req.body));
+});
+
+router.putAsync('/forms/:formId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.formId);
+
+    await forms.updateWithConsistencyCheck(req.context, entity);
+    return res.json();
+});
+
+router.deleteAsync('/forms/:formId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await forms.remove(req.context, castToInteger(req.params.formId));
+    return res.json();
+});
+
+router.postAsync('/forms-validate', passport.loggedIn, async (req, res) => {
+    return res.json(await forms.serverValidate(req.context, req.body));
+});
+
+
+module.exports = router;

server/routes/rest/import-runs.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const importRuns = require('../../models/import-runs');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+router.postAsync('/import-runs-table/:listId/:importId', passport.loggedIn, async (req, res) => {
+    return res.json(await importRuns.listDTAjax(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId), req.body));
+});
+
+router.postAsync('/import-run-failed-table/:listId/:importId/:importRunId', passport.loggedIn, async (req, res) => {
+    return res.json(await importRuns.listFailedDTAjax(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId), castToInteger(req.params.importRunId), req.body));
+});
+
+router.getAsync('/import-runs/:listId/:importId/:runId', passport.loggedIn, async (req, res) => {
+    const entity = await importRuns.getById(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId), castToInteger(req.params.runId));
+    return res.json(entity);
+});
+
+module.exports = router;

server/routes/rest/imports.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const imports = require('../../models/imports');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+const path = require('path');
+const files = require('../../models/files');
+const uploadedFilesDir = path.join(files.filesDir, 'uploaded');
+
+const multer = require('multer')({
+    dest: uploadedFilesDir
+});
+
+router.postAsync('/imports-table/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await imports.listDTAjax(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.getAsync('/imports/:listId/:importId', passport.loggedIn, async (req, res) => {
+    const entity = await imports.getById(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId), true);
+    entity.hash = imports.hash(entity);
+    return res.json(entity);
+});
+
+const fileFields = [
+    {name: 'csvFile', maxCount: 1}
+];
+
+router.postAsync('/imports/:listId', passport.loggedIn, passport.csrfProtection, multer.fields(fileFields), async (req, res) => {
+    const entity = JSON.parse(req.body.entity);
+
+    return res.json(await imports.create(req.context, castToInteger(req.params.listId), entity, req.files));
+});
+
+router.putAsync('/imports/:listId/:importId', passport.loggedIn, passport.csrfProtection, multer.fields(fileFields), async (req, res) => {
+    const entity = JSON.parse(req.body.entity);
+    entity.id = castToInteger(req.params.importId);
+
+    await imports.updateWithConsistencyCheck(req.context, castToInteger(req.params.listId), entity, req.files);
+    return res.json();
+});
+
+router.deleteAsync('/imports/:listId/:importId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await imports.remove(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId));
+    return res.json();
+});
+
+router.postAsync('/import-start/:listId/:importId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await imports.start(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId)));
+});
+
+router.postAsync('/import-stop/:listId/:importId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await imports.stop(req.context, castToInteger(req.params.listId), castToInteger(req.params.importId)));
+});
+
+module.exports = router;

server/routes/rest/lists.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const lists = require('../../models/lists');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/lists-table', passport.loggedIn, async (req, res) => {
+    return res.json(await lists.listDTAjax(req.context, req.body));
+});
+
+router.postAsync('/lists-with-segment-by-campaign-table/:campaignId', passport.loggedIn, async (req, res) => {
+    return res.json(await lists.listWithSegmentByCampaignDTAjax(req.context, castToInteger(req.params.campaignId), req.body));
+});
+
+router.getAsync('/lists/:listId', passport.loggedIn, async (req, res) => {
+    const list = await lists.getByIdWithListFields(req.context, castToInteger(req.params.listId));
+    list.hash = lists.hash(list);
+    return res.json(list);
+});
+
+router.postAsync('/lists', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await lists.create(req.context, req.body));
+});
+
+router.putAsync('/lists/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.listId);
+
+    await lists.updateWithConsistencyCheck(req.context, entity);
+    return res.json();
+});
+
+router.deleteAsync('/lists/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await lists.remove(req.context, castToInteger(req.params.listId));
+    return res.json();
+});
+
+
+module.exports = router;

server/routes/rest/mosaico-templates.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const mosaicoTemplates = require('../../models/mosaico-templates');
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.getAsync('/mosaico-templates/:mosaicoTemplateId', passport.loggedIn, async (req, res) => {
+    const mosaicoTemplate = await mosaicoTemplates.getById(req.context, castToInteger(req.params.mosaicoTemplateId));
+    mosaicoTemplate.hash = mosaicoTemplates.hash(mosaicoTemplate);
+    return res.json(mosaicoTemplate);
+});
+
+router.postAsync('/mosaico-templates', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await mosaicoTemplates.create(req.context, req.body));
+});
+
+router.putAsync('/mosaico-templates/:mosaicoTemplateId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const mosaicoTemplate = req.body;
+    mosaicoTemplate.id = castToInteger(req.params.mosaicoTemplateId);
+
+    await mosaicoTemplates.updateWithConsistencyCheck(req.context, mosaicoTemplate);
+    return res.json();
+});
+
+router.deleteAsync('/mosaico-templates/:mosaicoTemplateId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await mosaicoTemplates.remove(req.context, castToInteger(req.params.mosaicoTemplateId));
+    return res.json();
+});
+
+router.postAsync('/mosaico-templates-table', passport.loggedIn, async (req, res) => {
+    return res.json(await mosaicoTemplates.listDTAjax(req.context, req.body));
+});
+
+module.exports = router;

server/routes/rest/namespaces.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const _ = require('../../lib/translate')._;
+const namespaces = require('../../models/namespaces');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.getAsync('/namespaces/:nsId', passport.loggedIn, async (req, res) => {
+    const ns = await namespaces.getById(req.context, castToInteger(req.params.nsId));
+
+    ns.hash = namespaces.hash(ns);
+
+    return res.json(ns);
+});
+
+router.postAsync('/namespaces', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await namespaces.create(req.context, req.body));
+});
+
+router.putAsync('/namespaces/:nsId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const ns = req.body;
+    ns.id = castToInteger(req.params.nsId);
+
+    await namespaces.updateWithConsistencyCheck(req.context, ns);
+    return res.json();
+});
+
+router.deleteAsync('/namespaces/:nsId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await namespaces.remove(req.context, castToInteger(req.params.nsId));
+    return res.json();
+});
+
+router.getAsync('/namespaces-tree', passport.loggedIn, async (req, res) => {
+
+    const tree = await namespaces.listTree(req.context);
+
+    return res.json(tree);
+});
+
+
+module.exports = router;

server/routes/rest/report-templates.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const _ = require('../../lib/translate')._;
+const reportTemplates = require('../../models/report-templates');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.getAsync('/report-templates/:reportTemplateId', passport.loggedIn, async (req, res) => {
+    const reportTemplate = await reportTemplates.getById(req.context, castToInteger(req.params.reportTemplateId));
+    reportTemplate.hash = reportTemplates.hash(reportTemplate);
+    return res.json(reportTemplate);
+});
+
+router.postAsync('/report-templates', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await reportTemplates.create(req.context, req.body));
+});
+
+router.putAsync('/report-templates/:reportTemplateId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const reportTemplate = req.body;
+    reportTemplate.id = castToInteger(req.params.reportTemplateId);
+
+    await reportTemplates.updateWithConsistencyCheck(req.context, reportTemplate);
+    return res.json();
+});
+
+router.deleteAsync('/report-templates/:reportTemplateId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await reportTemplates.remove(req.context, castToInteger(req.params.reportTemplateId));
+    return res.json();
+});
+
+router.postAsync('/report-templates-table', passport.loggedIn, async (req, res) => {
+    return res.json(await reportTemplates.listDTAjax(req.context, req.body));
+});
+
+router.getAsync('/report-template-user-fields/:reportTemplateId', passport.loggedIn, async (req, res) => {
+    const userFields = await reportTemplates.getUserFieldsById(req.context, castToInteger(req.params.reportTemplateId));
+    return res.json(userFields);
+});
+
+module.exports = router;

server/routes/rest/reports.js

@@ -0,0 +1,97 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const _ = require('../../lib/translate')._;
+const reports = require('../../models/reports');
+const reportProcessor = require('../../lib/report-processor');
+const reportHelpers = require('../../lib/report-helpers');
+const shares = require('../../models/shares');
+const contextHelpers = require('../../lib/context-helpers');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+const fs = require('fs-extra');
+
+router.getAsync('/reports/:reportId', passport.loggedIn, async (req, res) => {
+    const report = await reports.getByIdWithTemplate(req.context, castToInteger(req.params.reportId));
+    report.hash = reports.hash(report);
+    return res.json(report);
+});
+
+router.postAsync('/reports', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await reports.create(req.context, req.body));
+});
+
+router.putAsync('/reports/:reportId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const report = req.body;
+    report.id = castToInteger(req.params.reportId);
+
+    await reports.updateWithConsistencyCheck(req.context, report);
+    return res.json();
+});
+
+router.deleteAsync('/reports/:reportId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await reports.remove(req.context, castToInteger(req.params.reportId));
+    return res.json();
+});
+
+router.postAsync('/reports-table', passport.loggedIn, async (req, res) => {
+    return res.json(await reports.listDTAjax(req.context, req.body));
+});
+
+router.postAsync('/report-start/:id', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const id = castToInteger(req.params.id);
+
+    await shares.enforceEntityPermission(req.context, 'report', id, 'execute');
+
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), id, false);
+    await shares.enforceEntityPermission(req.context, 'reportTemplate', report.report_template, 'execute');
+
+    await reportProcessor.start(id);
+    res.json();
+});
+
+router.postAsync('/report-stop/:id', async (req, res) => {
+    const id = castToInteger(req.params.id);
+
+    await shares.enforceEntityPermission(req.context, 'report', id, 'execute');
+
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), id, false);
+    await shares.enforceEntityPermission(req.context, 'reportTemplate', report.report_template, 'execute');
+
+    await reportProcessor.stop(id);
+    res.json();
+});
+
+router.getAsync('/report-content/:id', async (req, res) => {
+    const id = castToInteger(req.params.id);
+
+    await shares.enforceEntityPermission(req.context, 'report', id, 'viewContent');
+
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), id, false);
+    const file = reportHelpers.getReportContentFile(report);
+
+    if (await fs.pathExists(file)) {
+        res.sendFile(file);
+    } else {
+        res.send('');
+    }
+});
+
+router.getAsync('/report-output/:id', async (req, res) => {
+    const id = castToInteger(req.params.id);
+
+    await shares.enforceEntityPermission(req.context, 'report', id, 'viewOutput');
+
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), id, false);
+    const file = reportHelpers.getReportOutputFile(report);
+
+    if (await fs.pathExists(file)) {
+        res.sendFile(file);
+    } else {
+        res.send('');
+    }
+});
+
+
+module.exports = router;

server/routes/rest/segments.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const segments = require('../../models/segments');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/segments-table/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await segments.listDTAjax(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.getAsync('/segments/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await segments.listIdName(req.context, castToInteger(req.params.listId)));
+});
+
+router.getAsync('/segments/:listId/:segmentId', passport.loggedIn, async (req, res) => {
+    const segment = await segments.getById(req.context, castToInteger(req.params.listId), castToInteger(req.params.segmentId));
+    segment.hash = segments.hash(segment);
+    return res.json(segment);
+});
+
+router.postAsync('/segments/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await segments.create(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.putAsync('/segments/:listId/:segmentId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.segmentId);
+
+    await segments.updateWithConsistencyCheck(req.context, castToInteger(req.params.listId), entity);
+    return res.json();
+});
+
+router.deleteAsync('/segments/:listId/:segmentId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await segments.remove(req.context, castToInteger(req.params.listId), castToInteger(req.params.segmentId));
+    return res.json();
+});
+
+
+module.exports = router;

server/routes/rest/send-configurations.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const sendConfigurations = require('../../models/send-configurations');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.getAsync('/send-configurations-private/:sendConfigurationId', passport.loggedIn, async (req, res) => {
+    const sendConfiguration = await sendConfigurations.getById(req.context, castToInteger(req.params.sendConfigurationId), true, true);
+    sendConfiguration.hash = sendConfigurations.hash(sendConfiguration);
+    return res.json(sendConfiguration);
+});
+
+router.getAsync('/send-configurations-public/:sendConfigurationId', passport.loggedIn, async (req, res) => {
+    const sendConfiguration = await sendConfigurations.getById(req.context, castToInteger(req.params.sendConfigurationId), true, false);
+    sendConfiguration.hash = sendConfigurations.hash(sendConfiguration);
+    return res.json(sendConfiguration);
+});
+
+router.postAsync('/send-configurations', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await sendConfigurations.create(req.context, req.body));
+});
+
+router.putAsync('/send-configurations/:sendConfigurationId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const sendConfiguration = req.body;
+    sendConfiguration.id = castToInteger(req.params.sendConfigurationId);
+
+    await sendConfigurations.updateWithConsistencyCheck(req.context, sendConfiguration);
+    return res.json();
+});
+
+router.deleteAsync('/send-configurations/:sendConfigurationId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await sendConfigurations.remove(req.context, castToInteger(req.params.sendConfigurationId));
+    return res.json();
+});
+
+router.postAsync('/send-configurations-table', passport.loggedIn, async (req, res) => {
+    return res.json(await sendConfigurations.listDTAjax(req.context, req.body));
+});
+
+router.postAsync('/send-configurations-with-send-permission-table', passport.loggedIn, async (req, res) => {
+    return res.json(await sendConfigurations.listWithSendPermissionDTAjax(req.context, req.body));
+});
+
+module.exports = router;

server/routes/rest/settings.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const settings = require('../../models/settings');
+
+const router = require('../../lib/router-async').create();
+
+
+router.getAsync('/settings', passport.loggedIn, async (req, res) => {
+    const configItems = await settings.get(req.context);
+    configItems.hash = settings.hash(configItems);
+    return res.json(configItems);
+});
+
+router.putAsync('/settings', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const configItems = req.body;
+    await settings.set(req.context, configItems);
+    return res.json();
+});
+
+
+module.exports = router;

server/routes/rest/shares.js

@@ -0,0 +1,78 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const _ = require('../../lib/translate')._;
+const shares = require('../../models/shares');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+router.postAsync('/shares-table-by-entity/:entityTypeId/:entityId', passport.loggedIn, async (req, res) => {
+    return res.json(await shares.listByEntityDTAjax(req.context, req.params.entityTypeId, castToInteger(req.params.entityId), req.body));
+});
+
+router.postAsync('/shares-table-by-user/:entityTypeId/:userId', passport.loggedIn, async (req, res) => {
+    return res.json(await shares.listByUserDTAjax(req.context, req.params.entityTypeId, castToInteger(req.params.userId), req.body));
+});
+
+router.postAsync('/shares-unassigned-users-table/:entityTypeId/:entityId', passport.loggedIn, async (req, res) => {
+    return res.json(await shares.listUnassignedUsersDTAjax(req.context, req.params.entityTypeId, castToInteger(req.params.entityId), req.body));
+});
+
+router.postAsync('/shares-roles-table/:entityTypeId', passport.loggedIn, async (req, res) => {
+    return res.json(await shares.listRolesDTAjax(req.params.entityTypeId, req.body));
+});
+
+router.putAsync('/shares', passport.loggedIn, async (req, res) => {
+    const body = req.body;
+    await shares.assign(req.context, body.entityTypeId, body.entityId, body.userId, body.role);
+
+    return res.json();
+});
+
+/*
+ Checks if entities with a given permission exist.
+
+ Accepts format:
+ {
+   XXX1: {
+     entityTypeId: ...
+     requiredOperations: [ ... ]
+   },
+
+   XXX2: {
+     entityTypeId: ...
+     requiredOperations: [ ... ]
+   }
+ }
+
+ Returns:
+ {
+   XXX1: true
+   XXX2: false
+ }
+ */
+router.postAsync('/permissions-check', passport.loggedIn, async (req, res) => {
+    const body = req.body;
+    const result = {};
+
+    for (const reqKey in body) {
+        if (body[reqKey].entityId) {
+            result[reqKey] = await shares.checkEntityPermission(req.context, body[reqKey].entityTypeId, body[reqKey].entityId, body[reqKey].requiredOperations);
+        } else {
+            result[reqKey] = await shares.checkTypePermission(req.context, body[reqKey].entityTypeId, body[reqKey].requiredOperations);
+        }
+    }
+
+    return res.json(result);
+});
+
+router.postAsync('/permissions-rebuild', passport.loggedIn, async (req, res) => {
+    shares.enforceGlobalPermission(req.context, 'rebuildPermissions');
+    await shares.rebuildPermissions();
+    return res.json(result);
+});
+
+
+
+module.exports = router;

server/routes/rest/subscriptions.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const subscriptions = require('../../models/subscriptions');
+const { SubscriptionSource } = require('../../../shared/lists');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/subscriptions-table/:listId/:segmentId?', passport.loggedIn, async (req, res) => {
+    return res.json(await subscriptions.listDTAjax(req.context, castToInteger(req.params.listId), req.params.segmentId ? castToInteger(req.params.segmentId) : null, req.body));
+});
+
+router.postAsync('/subscriptions-test-user-table/:listCid', passport.loggedIn, async (req, res) => {
+    return res.json(await subscriptions.listTestUsersDTAjax(req.context, req.params.listCid, req.body));
+});
+
+router.getAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, async (req, res) => {
+    const entity = await subscriptions.getById(req.context, castToInteger(req.params.listId), castToInteger(req.params.subscriptionId));
+    entity.hash = await subscriptions.hashByList(castToInteger(req.params.listId), entity);
+    return res.json(entity);
+});
+
+router.postAsync('/subscriptions/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await subscriptions.create(req.context, castToInteger(req.params.listId), req.body, SubscriptionSource.ADMIN_FORM));
+});
+
+router.putAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.subscriptionId);
+
+    await subscriptions.updateWithConsistencyCheck(req.context, castToInteger(req.params.listId), entity, SubscriptionSource.ADMIN_FORM);
+    return res.json();
+});
+
+router.deleteAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await subscriptions.remove(req.context, castToInteger(req.params.listId), castToInteger(req.params.subscriptionId));
+    return res.json();
+});
+
+router.postAsync('/subscriptions-validate/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await subscriptions.serverValidate(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.postAsync('/subscriptions-unsubscribe/:listId/:subscriptionId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await subscriptions.unsubscribeByIdAndGet(req.context, castToInteger(req.params.listId), castToInteger(req.params.subscriptionId));
+    return res.json();
+});
+
+
+module.exports = router;

server/routes/rest/templates.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const templates = require('../../models/templates');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+const CampaignSender = require('../../lib/campaign-sender');
+
+
+router.getAsync('/templates/:templateId', passport.loggedIn, async (req, res) => {
+    const template = await templates.getById(req.context, castToInteger(req.params.templateId));
+    template.hash = templates.hash(template);
+    return res.json(template);
+});
+
+router.postAsync('/templates', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await templates.create(req.context, req.body));
+});
+
+router.putAsync('/templates/:templateId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const template = req.body;
+    template.id = castToInteger(req.params.templateId);
+
+    await templates.updateWithConsistencyCheck(req.context, template);
+    return res.json();
+});
+
+router.deleteAsync('/templates/:templateId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await templates.remove(req.context, castToInteger(req.params.templateId));
+    return res.json();
+});
+
+router.postAsync('/templates-table', passport.loggedIn, async (req, res) => {
+    return res.json(await templates.listDTAjax(req.context, req.body));
+});
+
+router.postAsync('/template-test-send', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const data = req.body;
+    const result = await CampaignSender.testSend(req.context, data.listCid, data.subscriptionCid, data.campaignId, data.sendConfigurationId, data.html, data.text);
+    return res.json(result);
+});
+
+module.exports = router;

server/routes/rest/triggers.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const triggers = require('../../models/triggers');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.postAsync('/triggers-by-campaign-table/:campaignId', passport.loggedIn, async (req, res) => {
+    return res.json(await triggers.listByCampaignDTAjax(req.context, castToInteger(req.params.campaignId), req.body));
+});
+
+router.postAsync('/triggers-by-list-table/:listId', passport.loggedIn, async (req, res) => {
+    return res.json(await triggers.listByListDTAjax(req.context, castToInteger(req.params.listId), req.body));
+});
+
+router.getAsync('/triggers/:campaignId/:triggerId', passport.loggedIn, async (req, res) => {
+    const entity = await triggers.getById(req.context, castToInteger(req.params.campaignId), req.params.triggerId);
+    entity.hash = triggers.hash(entity);
+    return res.json(entity);
+});
+
+router.postAsync('/triggers/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await triggers.create(req.context, castToInteger(req.params.campaignId), req.body));
+});
+
+router.putAsync('/triggers/:campaignId/:triggerId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = castToInteger(req.params.triggerId);
+
+    await triggers.updateWithConsistencyCheck(req.context, castToInteger(req.params.campaignId), entity);
+    return res.json();
+});
+
+router.deleteAsync('/triggers/:campaignId/:triggerId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await triggers.remove(req.context, castToInteger(req.params.campaignId), castToInteger(req.params.triggerId));
+    return res.json();
+});
+
+module.exports = router;

server/routes/rest/users.js

@@ -0,0 +1,45 @@
+'use strict';
+
+const config = require('config');
+const passport = require('../../lib/passport');
+const _ = require('../../lib/translate')._;
+const users = require('../../models/users');
+const shares = require('../../models/shares');
+
+const router = require('../../lib/router-async').create();
+const {castToInteger} = require('../../lib/helpers');
+
+
+router.getAsync('/users/:userId', passport.loggedIn, async (req, res) => {
+    const user = await users.getById(req.context, castToInteger(req.params.userId));
+    user.hash = users.hash(user);
+    return res.json(user);
+});
+
+router.postAsync('/users', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await users.create(req.context, req.body));
+});
+
+router.putAsync('/users/:userId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const user = req.body;
+    user.id = castToInteger(req.params.userId);
+
+    await users.updateWithConsistencyCheck(req.context, user);
+    return res.json();
+});
+
+router.deleteAsync('/users/:userId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await users.remove(req.context, castToInteger(req.params.userId));
+    return res.json();
+});
+
+router.postAsync('/users-validate', passport.loggedIn, async (req, res) => {
+    return res.json(await users.serverValidate(req.context, req.body));
+});
+
+router.postAsync('/users-table', passport.loggedIn, async (req, res) => {
+    return res.json(await users.listDTAjax(req.context, req.body));
+});
+
+
+module.exports = router;

server/routes/sandboxed-ckeditor.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const routerFactory = require('../lib/router-async');
+const passport = require('../lib/passport');
+const clientHelpers = require('../lib/client-helpers');
+const users = require('../models/users');
+
+const files = require('../models/files');
+const fileHelpers = require('../lib/file-helpers');
+
+const templates = require('../models/templates');
+
+const contextHelpers = require('../lib/context-helpers');
+
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const { AppType } = require('../../shared/app');
+
+
+users.registerRestrictedAccessTokenMethod('ckeditor', async ({entityTypeId, entityId}) => {
+    if (entityTypeId === 'template') {
+        const tmpl = await templates.getById(contextHelpers.getAdminContext(), entityId, false);
+
+        if (tmpl.type === 'ckeditor4') {
+            return {
+                permissions: {
+                    'template': {
+                        [entityId]: new Set(['manageFiles', 'view'])
+                    }
+                }
+            };
+        }
+    }
+});
+
+
+function getRouter(appType) {
+    const router = routerFactory.create();
+
+    if (appType === AppType.SANDBOXED) {
+        router.getAsync('/editor', passport.csrfProtection, async (req, res) => {
+            const mailtrainConfig = await clientHelpers.getAnonymousConfig(req.context, appType);
+
+            res.render('ckeditor/root', {
+                layout: 'ckeditor/layout',
+                reactCsrfToken: req.csrfToken(),
+                mailtrainConfig: JSON.stringify(mailtrainConfig),
+                scriptFiles: [
+                    getSandboxUrl('mailtrain/ckeditor-root.js')
+                ],
+                publicPath: getSandboxUrl()
+            });
+        });
+    }
+
+    return router;
+}
+
+module.exports.getRouter = getRouter;

server/routes/sandboxed-codeeditor.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const routerFactory = require('../lib/router-async');
+const passport = require('../lib/passport');
+const clientHelpers = require('../lib/client-helpers');
+const users = require('../models/users');
+
+const files = require('../models/files');
+const fileHelpers = require('../lib/file-helpers');
+
+const templates = require('../models/templates');
+
+const contextHelpers = require('../lib/context-helpers');
+
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const { AppType } = require('../../shared/app');
+
+
+users.registerRestrictedAccessTokenMethod('codeeditor', async ({entityTypeId, entityId}) => {
+    if (entityTypeId === 'template') {
+        const tmpl = await templates.getById(contextHelpers.getAdminContext(), entityId, false);
+
+        if (tmpl.type === 'codeeditor') {
+            return {
+                permissions: {
+                    'template': {
+                        [entityId]: new Set(['manageFiles', 'view'])
+                    }
+                }
+            };
+        }
+    }
+});
+
+
+function getRouter(appType) {
+    const router = routerFactory.create();
+
+    if (appType === AppType.SANDBOXED) {
+        router.getAsync('/editor', passport.csrfProtection, async (req, res) => {
+            const mailtrainConfig = await clientHelpers.getAnonymousConfig(req.context, appType);
+
+            res.render('ckeditor/root', {
+                layout: 'ckeditor/layout',
+                reactCsrfToken: req.csrfToken(),
+                mailtrainConfig: JSON.stringify(mailtrainConfig),
+                scriptFiles: [
+                    getSandboxUrl('mailtrain/codeeditor-root.js')
+                ],
+                publicPath: getSandboxUrl()
+            });
+        });
+    }
+
+    return router;
+}
+
+module.exports.getRouter = getRouter;

server/routes/sandboxed-grapesjs.js

@@ -0,0 +1,65 @@
+'use strict';
+
+const routerFactory = require('../lib/router-async');
+const passport = require('../lib/passport');
+const clientHelpers = require('../lib/client-helpers');
+const users = require('../models/users');
+
+const files = require('../models/files');
+const fileHelpers = require('../lib/file-helpers');
+
+const templates = require('../models/templates');
+
+const contextHelpers = require('../lib/context-helpers');
+
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const { AppType } = require('../../shared/app');
+
+
+users.registerRestrictedAccessTokenMethod('grapesjs', async ({entityTypeId, entityId}) => {
+    if (entityTypeId === 'template') {
+        const tmpl = await templates.getById(contextHelpers.getAdminContext(), entityId, false);
+
+        if (tmpl.type === 'grapesjs') {
+            return {
+                permissions: {
+                    'template': {
+                        [entityId]: new Set(['viewFiles', 'manageFiles', 'view'])
+                    }
+                }
+            };
+        }
+    }
+});
+
+
+function getRouter(appType) {
+    const router = routerFactory.create();
+    
+    if (appType === AppType.SANDBOXED) {
+        router.getAsync('/editor', passport.csrfProtection, async (req, res) => {
+            const mailtrainConfig = await clientHelpers.getAnonymousConfig(req.context, appType);
+
+            res.render('grapesjs/root', {
+                layout: 'grapesjs/layout',
+                reactCsrfToken: req.csrfToken(),
+                mailtrainConfig: JSON.stringify(mailtrainConfig),
+                scriptFiles: [
+                    getSandboxUrl('mailtrain/grapesjs-root.js')
+                ],
+                publicPath: getSandboxUrl()
+            });
+        });
+
+        fileHelpers.installUploadHandler(router, '/upload/:type/:entityId', files.ReplacementBehavior.RENAME, null, 'file', resp => {
+            return {
+                data: resp.files.map( f => ({type: 'image', src: f.url}) )
+            };
+        });
+
+    }
+
+    return router;
+}
+
+module.exports.getRouter = getRouter;

server/routes/sandboxed-mosaico.js

@@ -0,0 +1,252 @@
+'use strict';
+
+const config = require('config');
+const path = require('path');
+const express = require('express');
+const routerFactory = require('../lib/router-async');
+const passport = require('../lib/passport');
+const clientHelpers = require('../lib/client-helpers');
+const gm = require('gm').subClass({
+    imageMagick: true
+});
+const users = require('../models/users');
+
+const fs = require('fs-extra')
+
+const files = require('../models/files');
+const fileHelpers = require('../lib/file-helpers');
+
+const templates = require('../models/templates');
+const mosaicoTemplates = require('../models/mosaico-templates');
+
+const contextHelpers = require('../lib/context-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const { base } = require('../../shared/templates');
+const { AppType } = require('../../shared/app');
+
+const {castToInteger} = require('../lib/helpers');
+
+
+users.registerRestrictedAccessTokenMethod('mosaico', async ({entityTypeId, entityId}) => {
+    if (entityTypeId === 'template') {
+        const tmpl = await templates.getById(contextHelpers.getAdminContext(), entityId, false);
+
+        if (tmpl.type === 'mosaico') {
+            return {
+                permissions: {
+                    'template': {
+                        [entityId]: new Set(['viewFiles', 'manageFiles', 'view'])
+                    },
+                    'mosaicoTemplate': {
+                        [tmpl.data.mosaicoTemplate]: new Set(['view'])
+                    }
+                }
+            };
+        }
+    }
+});
+
+
+async function placeholderImage(width, height) {
+    const magick = gm(width, height, '#707070');
+    const streamAsync = bluebird.promisify(magick.stream.bind(magick));
+
+    const size = 40;
+    let x = 0;
+    let y = 0;
+
+    // stripes
+    while (y < height) {
+        magick
+            .fill('#808080')
+            .drawPolygon([x, y], [x + size, y], [x + size * 2, y + size], [x + size * 2, y + size * 2])
+            .drawPolygon([x, y + size], [x + size, y + size * 2], [x, y + size * 2]);
+        x = x + size * 2;
+        if (x > width) {
+            x = 0;
+            y = y + size * 2;
+        }
+    }
+
+    // text
+    magick
+        .fill('#B0B0B0')
+        .fontSize(20)
+        .drawText(0, 0, width + ' x ' + height, 'center');
+
+    const stream = await streamAsync('png');
+
+    return {
+        format: 'png',
+        stream
+    };
+}
+
+async function resizedImage(filePath, method, width, height) {
+    const magick = gm(filePath);
+    const streamAsync = bluebird.promisify(magick.stream.bind(magick));
+    const formatAsync = bluebird.promisify(magick.format.bind(magick));
+
+    const format = (await formatAsync()).toLowerCase();
+
+    if (method === 'resize') {
+        magick
+            .autoOrient()
+            .resize(width, height);
+    } else if (method === 'cover') {
+        magick
+            .autoOrient()
+            .resize(width, height + '^')
+            .gravity('Center')
+            .extent(width, height + '>');
+    } else {
+        throw new Error(`Method ${method} not supported`);
+    }
+
+    const stream = await streamAsync();
+
+    return {
+        format,
+        stream
+    };
+}
+
+function sanitizeSize(val, min, max, defaultVal, allowNull) {
+    if (val === 'null' && allowNull) {
+        return null;
+    }
+    val = Number(val) || defaultVal;
+    val = Math.max(min, val);
+    val = Math.min(max, val);
+    return val;
+}
+
+
+
+function getRouter(appType) {
+    const router = routerFactory.create();
+    
+    if (appType === AppType.SANDBOXED) {
+        router.getAsync('/templates/:mosaicoTemplateId/index.html', passport.loggedIn, async (req, res) => {
+            const tmpl = await mosaicoTemplates.getById(req.context, castToInteger(req.params.mosaicoTemplateId));
+
+            res.set('Content-Type', 'text/html');
+            res.send(base(tmpl.data.html, getTrustedUrl(), getSandboxUrl('', req.context), getPublicUrl()));
+        });
+
+        // Mosaico looks for block thumbnails in edres folder relative to index.html of the template. We respond to such requests here.
+        router.getAsync('/templates/:mosaicoTemplateId/edres/:fileName', async (req, res, next) => {
+            try {
+                const file = await files.getFileByOriginalName(contextHelpers.getAdminContext(), 'mosaicoTemplate', 'block', castToInteger(req.params.mosaicoTemplateId), req.params.fileName);
+                res.type(file.mimetype);
+                return res.download(file.path, file.name);
+            } catch (err) {
+                if (err instanceof interoperableErrors.NotFoundError) {
+                    next();
+                } else {
+                    throw err;
+                }
+            }
+        });
+
+        // This is a fallback to versafix-1 if the block thumbnail is not defined by the template
+        router.use('/templates/:mosaicoTemplateId/edres', express.static(path.join(__dirname, '..', 'client', 'static', 'mosaico', 'templates', 'versafix-1', 'edres')));
+
+
+        fileHelpers.installUploadHandler(router, '/upload/:type/:entityId', files.ReplacementBehavior.RENAME, null, 'file', resp => {
+            return {
+                files: resp.files.map(f => ({name: f.name, url: f.url, size: f.size, thumbnailUrl: f.thumbnailUrl}))
+            };
+        });
+
+        router.getAsync('/upload/:type/:entityId', passport.loggedIn, async (req, res) => {
+            const id = castToInteger(req.params.entityId);
+
+            const entries = await files.list(req.context, req.params.type, 'file', id);
+
+            const filesOut = [];
+            for (const entry of entries) {
+                filesOut.push({
+                    name: entry.originalname,
+                    url: files.getFileUrl(req.context, req.params.type, 'file', id, entry.filename),
+                    size: entry.size,
+                    thumbnailUrl: files.getFileUrl(req.context, req.params.type, 'file', id, entry.filename) // TODO - use smaller thumbnails
+                })
+            }
+
+            res.json({
+                files: filesOut
+            });
+        });
+
+        router.getAsync('/editor', passport.csrfProtection, async (req, res) => {
+            const mailtrainConfig = await clientHelpers.getAnonymousConfig(req.context, appType);
+
+            let languageStrings = null;
+            if (config.language && config.language !== 'en') {
+                const lang = config.language.split('_')[0];
+                try {
+                    const file = path.join(__dirname, '..', 'client', 'static', 'mosaico', 'lang', 'mosaico-' + lang + '.json');
+                    languageStrings = await fs.readFile(file, 'utf8');
+                } catch (err) {
+                }
+            }
+
+            res.render('mosaico/root', {
+                layout: 'mosaico/layout',
+                editorConfig: config.mosaico,
+                languageStrings: languageStrings,
+                reactCsrfToken: req.csrfToken(),
+                mailtrainConfig: JSON.stringify(mailtrainConfig),
+                scriptFiles: [
+                    getSandboxUrl('mailtrain/mosaico-root.js')
+                ],
+                publicPath: getSandboxUrl()
+            });
+        });
+
+    } else if (appType === AppType.TRUSTED || appType === AppType.PUBLIC) { // Mosaico editor loads the images from TRUSTED endpoint. This is hard to change because the index.html has to come from TRUSTED.
+                                                                            // So we serve /mosaico/img under both endpoints. There is no harm in it.
+        router.getAsync('/img', async (req, res) => {
+            const method = req.query.method;
+            const params = req.query.params;
+            let [width, height] = params.split(',');
+            let image;
+
+
+            // FIXME - cache the generated files !!!
+
+            if (method === 'placeholder') {
+                width = sanitizeSize(width, 1, 2048, 600, false);
+                height = sanitizeSize(height, 1, 2048, 300, false);
+                image = await placeholderImage(width, height);
+
+            } else {
+                width = sanitizeSize(width, 1, 2048, 600, false);
+                height = sanitizeSize(height, 1, 2048, 300, true);
+
+                let filePath;
+                const url = req.query.src;
+
+                const mosaicoLegacyUrlPrefix = getTrustedUrl(`mosaico/uploads/`);
+                if (url.startsWith(mosaicoLegacyUrlPrefix)) {
+                    filePath = path.join(__dirname, '..', 'client', 'static' , 'mosaico', 'uploads', url.substring(mosaicoLegacyUrlPrefix.length));
+                } else {
+                    const file = await files.getFileByUrl(contextHelpers.getAdminContext(), url);
+                    filePath = file.path;
+                }
+
+                image = await resizedImage(filePath, method, width, height);
+            }
+
+            res.set('Content-Type', 'image/' + image.format);
+            image.stream.pipe(res);
+        });
+    }
+
+    return router;
+}
+
+module.exports.getRouter = getRouter;

server/routes/subscription.js

@@ -0,0 +1,692 @@
+'use strict';
+
+const log = require('../lib/log');
+const config = require('config');
+const router = require('../lib/router-async').create();
+const confirmations = require('../models/confirmations');
+const subscriptions = require('../models/subscriptions');
+const lists = require('../models/lists');
+const fields = require('../models/fields');
+const shares = require('../models/shares');
+const settings = require('../models/settings');
+const { tUI } = require('../lib/translate');
+const contextHelpers = require('../lib/context-helpers');
+const forms = require('../models/forms');
+const {getTrustedUrl} = require('../lib/urls');
+const bluebird = require('bluebird');
+
+const { SubscriptionStatus, SubscriptionSource } = require('../../shared/lists');
+
+const openpgp = require('openpgp');
+const cors = require('cors');
+const cache = require('memory-cache');
+const geoip = require('geoip-ultralight');
+const passport = require('../lib/passport');
+
+const tools = require('../lib/tools');
+const mailHelpers = require('../lib/subscription-mail-helpers');
+
+const interoperableErrors = require('../../shared/interoperable-errors');
+
+const { cleanupFromPost } = require('../lib/helpers');
+
+const originWhitelist = config.cors && config.cors.origins || [];
+
+const corsOptions = {
+    allowedHeaders: ['Content-Type', 'Origin', 'Accept', 'X-Requested-With'],
+    methods: ['GET', 'POST'],
+    optionsSuccessStatus: 200, // IE11 chokes on 204
+    origin: (origin, callback) => {
+        if (originWhitelist.includes(origin)) {
+            callback(null, true);
+        } else {
+            const err = new Error('Not allowed by CORS');
+            err.status = 403;
+            callback(err);
+        }
+    }
+};
+
+const corsOrCsrfProtection = (req, res, next) => {
+    if (req.get('X-Requested-With') === 'XMLHttpRequest') {
+        cors(corsOptions)(req, res, next);
+    } else {
+        passport.csrfProtection(req, res, next);
+    }
+};
+
+async function takeConfirmationAndValidate(req, action, errorFactory) {
+    const confirmation = await confirmations.takeConfirmation(req.params.cid);
+
+    if (!confirmation || confirmation.action !== action) {
+        throw errorFactory();
+    }
+
+    return confirmation;
+}
+
+async function injectCustomFormData(customFormId, viewKey, data) {
+    function sortAndFilterCustomFieldsBy(key) {
+        data.customFields = data.customFields.filter(fld => fld[key] !== null);
+        data.customFields.sort((a, b) => a[key] - b[key]);
+    }
+
+    if (viewKey === 'web_subscribe') {
+        sortAndFilterCustomFieldsBy('order_subscribe');
+    } else if (viewKey === 'web_manage') {
+        sortAndFilterCustomFieldsBy('order_manage');
+    }
+
+    if (!customFormId) {
+        data.formInputStyle = '@import url(/static/subscription/form-input-style.css);';
+        return;
+    }
+
+    const form = await forms.getById(contextHelpers.getAdminContext(), customFormId);
+
+    data.template.template = form[viewKey] || data.template.template;
+    data.template.layout = form.layout || data.template.layout;
+    data.formInputStyle = form.formInputStyle || '@import url(/static/subscription/form-input-style.css);';
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['uaCode']);
+
+    data.uaCode = configItems.uaCode;
+    data.customSubscriptionScripts = config.customSubscriptionScripts || [];
+}
+
+async function captureFlashMessages(res) {
+    const renderAsync = bluebird.promisify(res.render.bind(res));
+    return await renderAsync('subscription/capture-flash-messages', { layout: null });
+}
+
+
+router.getAsync('/confirm/subscribe/:cid', async (req, res) => {
+    const confirmation = await takeConfirmationAndValidate(req, 'subscribe', () => new interoperableErrors.InvalidConfirmationForSubscriptionError('Request invalid or already completed. If your subscription request is still pending, please subscribe again.'));
+    const data = confirmation.data;
+
+    const meta = {
+        ip: confirmation.ip,
+        country: geoip.lookupCountry(confirmation.ip) || null,
+        updateOfUnsubscribedAllowed: true
+    };
+
+    const subscription = data.subscriptionData;
+    subscription.email = data.email;
+    subscription.status = SubscriptionStatus.SUBSCRIBED;
+
+    try {
+        await subscriptions.create(contextHelpers.getAdminContext(), confirmation.list, subscription,  SubscriptionSource.SUBSCRIPTION_FORM, meta);
+    } catch (err) {
+        if (err instanceof interoperableErrors.DuplicitEmailError) {
+            throw new interoperableErrors.DuplicitEmailError('Subscription already present'); // This is here to provide some meaningful error message.
+        } else {
+            throw err;
+        }
+    }
+
+    const list = await lists.getById(contextHelpers.getAdminContext(), confirmation.list);
+    subscription.cid = meta.cid;
+    await mailHelpers.sendSubscriptionConfirmed(req.language, list, subscription.email, subscription);
+
+    res.redirect('/subscription/' + encodeURIComponent(list.cid) + '/subscribed-notice');
+});
+
+
+router.getAsync('/confirm/change-address/:cid', async (req, res) => {
+    const confirmation = await takeConfirmationAndValidate(req, 'change-address', () => new interoperableErrors.InvalidConfirmationForAddressChangeError('Request invalid or already completed. If your address change request is still pending, please change the address again.'));
+    const list = await lists.getById(contextHelpers.getAdminContext(), confirmation.list);
+    const data = confirmation.data;
+
+    const subscription = await subscriptions.updateAddressAndGet(contextHelpers.getAdminContext(), list.id, data.subscriptionId, data.emailNew);
+
+    await mailHelpers.sendSubscriptionConfirmed(req.language, list, data.emailNew, subscription);
+
+    req.flash('info', tUI('subscription.emailChanged', req.language));
+    res.redirect('/subscription/' + encodeURIComponent(list.cid) + '/manage/' + subscription.cid);
+});
+
+
+router.getAsync('/confirm/unsubscribe/:cid', async (req, res) => {
+    const confirmation = await takeConfirmationAndValidate(req, 'unsubscribe', () => new interoperableErrors.InvalidConfirmationForUnsubscriptionError('Request invalid or already completed. If your unsubscription request is still pending, please unsubscribe again.'));
+    const list = await lists.getById(contextHelpers.getAdminContext(), confirmation.list);
+    const data = confirmation.data;
+
+    const subscription = await subscriptions.unsubscribeByCidAndGet(contextHelpers.getAdminContext(), list.id, data.subscriptionCid, data.campaignCid);
+
+    await mailHelpers.sendUnsubscriptionConfirmed(req.language, list, subscription.email, subscription);
+
+    res.redirect('/subscription/' + encodeURIComponent(list.cid) + '/unsubscribed-notice');
+});
+
+
+async function _renderSubscribe(req, res, list, subscription) {
+    const data = {};
+    data.email = subscription && subscription.email;
+    data.layout = 'subscription/layout';
+    data.title = list.name;
+    data.cid = list.cid;
+    data.csrfToken = req.csrfToken();
+
+    data.customFields = await fields.forHbs(contextHelpers.getAdminContext(), list.id, subscription);
+    data.useEditor = true;
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey']);
+    data.hasPubkey = !!configItems.pgpPrivateKey;
+
+    data.template = {
+        template: 'subscription/web-subscribe.mjml.hbs',
+        layout: 'subscription/layout.mjml.hbs',
+        type: 'mjml'
+    };
+
+    await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-subscribe', data);
+
+    const htmlRenderer = await tools.getTemplate(data.template);
+
+    data.isWeb = true;
+    data.needsJsWarning = true;
+
+    data.flashMessages = await captureFlashMessages(res);
+
+    const result = htmlRenderer(data);
+
+    res.send(result);
+}
+
+router.getAsync('/:cid', passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+
+    if (!list.public_subscribe) {
+        shares.throwPermissionDenied();
+    }
+
+    const ucid = req.query.cid;
+
+    let subscription;
+    if (ucid) {
+        try {
+            subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, ucid);
+
+            if (subscription.status === SubscriptionStatus.SUBSCRIBED) {
+                subscription = null;
+            }
+        } catch (err) {
+            if (err instanceof interoperableErrors.NotFoundError) {
+            } else {
+                throw err;
+            }
+        }
+    }
+
+    await _renderSubscribe(req, res, list, subscription);
+});
+
+
+router.options('/:cid/subscribe', cors(corsOptions));
+
+router.postAsync('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, async (req, res) => {
+    if (req.xhr) {
+        req.needsAPIJSONResponse = true;
+    }
+
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+
+    if (!list.public_subscribe) {
+        shares.throwPermissionDenied();
+    }
+
+    const subscriptionData = await fields.fromPost(contextHelpers.getAdminContext(), list.id, req.body);
+
+    const email = cleanupFromPost(req.body.EMAIL);
+
+    if (!email) {
+        if (req.xhr) {
+            throw new Error('Email address not set');
+        }
+
+        req.flash('danger', tUI('subscription.addressNotSet', req.language));
+        return await _renderSubscribe(req, res, list, subscriptionData);
+    }
+
+    const emailErr = await tools.validateEmail(email);
+    if (emailErr) {
+        const errMsg = tools.validateEmailGetMessage(emailErr, email);
+
+        if (req.xhr) {
+            throw new Error(errMsg);
+        }
+
+        req.flash('danger', errMsg);
+        subscriptionData.email = email;
+        return await _renderSubscribe(req, res, list, subscriptionData);
+    }
+
+    // Check if the subscriber seems legit. This is a really simple check, the only requirement is that
+    // the subscriber has JavaScript turned on and thats it. If Mailtrain gets more targeted then this
+    // simple check should be replaced with an actual captcha
+    let subTime = Number(req.body.sub) || 0;
+    // allow clock skew 24h in the past and 24h to the future
+    let subTimeTest = !!(subTime > Date.now() - 24 * 3600 * 1000 && subTime < Date.now() + 24 * 3600 * 1000);
+    let addressTest = !req.body.address;
+    let testsPass = subTimeTest && addressTest;
+
+    let existingSubscription;
+    try {
+        existingSubscription = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, email);
+    } catch (err) {
+        if (err instanceof interoperableErrors.NotFoundError) {
+        } else {
+            throw err;
+        }
+    }
+
+    if (existingSubscription && existingSubscription.status === SubscriptionStatus.SUBSCRIBED) {
+        await mailHelpers.sendAlreadySubscribed(req.language, list, email, existingSubscription);
+        res.redirect('/subscription/' + encodeURIComponent(req.params.cid) + '/confirm-subscription-notice');
+
+    } else {
+        const data = {
+            email,
+            subscriptionData
+        };
+
+        const confirmCid = await confirmations.addConfirmation(list.id, 'subscribe', req.ip, data);
+
+        if (!testsPass) {
+            log.info('Subscription', 'Confirmation message for %s marked to be skipped (%s)', email, JSON.stringify(data));
+        } else {
+            await mailHelpers.sendConfirmSubscription(req.language, list, email, confirmCid, subscriptionData);
+        }
+
+        if (req.xhr) {
+            return res.status(200).json({
+                msg: tUI('subscription.confirmSubscription', req.language)
+            });
+        }
+        res.redirect('/subscription/' + encodeURIComponent(req.params.cid) + '/confirm-subscription-notice');
+    }
+});
+
+
+router.options('/:cid/widget', cors(corsOptions));
+
+router.getAsync('/:cid/widget', cors(corsOptions), async (req, res) => {
+    req.needsAPIJSONResponse = true;
+
+    const cached = cache.get(req.path);
+    if (cached) {
+        return res.status(200).json(cached);
+    }
+
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey']);
+
+    const data = {
+        title: list.name,
+        cid: list.cid,
+        publicKeyUrl: getTrustedUrl('subscription/publickey'),
+        subscribeUrl: getTrustedUrl(`subscription/${list.cid}/subscribe`),
+        hasPubkey: !!configItems.pgpPrivateKey,
+        customFields: await fields.forHbs(contextHelpers.getAdminContext(), list.id),
+        template: {},
+        layout: null,
+    };
+
+    await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-subscribe', data);
+
+    const renderAsync = bluebird.promisify(res.render);
+    const html = await renderAsync('subscription/widget-subscribe', data);
+
+    const response = {
+        data: {
+            title: data.title,
+            cid: data.cid,
+            html
+        }
+    };
+
+    cache.put(req.path, response, 30000); // ms
+    res.status(200).json(response);
+});
+
+
+
+router.getAsync('/:lcid/manage/:ucid', passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
+
+    const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid);
+
+    if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+        throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+    }
+
+    const data = {};
+    data.email = subscription.email;
+    data.cid = subscription.cid;
+    data.lcid = req.params.lcid;
+    data.title = list.name;
+    data.csrfToken = req.csrfToken();
+    data.layout = 'data/layout';
+
+    data.customFields = await fields.forHbs(contextHelpers.getAdminContext(), list.id, subscription);
+
+    data.useEditor = true;
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey']);
+    data.hasPubkey = !!configItems.pgpPrivateKey;
+
+    data.template = {
+        template: 'subscription/web-manage.mjml.hbs',
+        layout: 'subscription/layout.mjml.hbs',
+        type: 'mjml'
+    };
+
+    await injectCustomFormData(req.query.fid || list.default_form, 'data/web-manage', data);
+
+    const htmlRenderer = await tools.getTemplate(data.template);
+
+    data.isWeb = true;
+    data.needsJsWarning = true;
+    data.isManagePreferences = true;
+    data.flashMessages = await captureFlashMessages(res);
+
+    res.send(htmlRenderer(data));
+});
+
+router.postAsync('/:lcid/manage', passport.parseForm, passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
+
+    try {
+        const subscriptionData = await fields.fromPost(contextHelpers.getAdminContext(), list.id, req.body);
+        await subscriptions.updateManaged(contextHelpers.getAdminContext(), list.id, req.body.cid, subscriptionData);
+    } catch (err) {
+        if (err instanceof interoperableErrors.NotFoundError) {
+            throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+        } else {
+            throw err;
+        }
+    }
+
+    res.redirect('/subscription/' + encodeURIComponent(req.params.lcid) + '/updated-notice');
+});
+
+router.getAsync('/:lcid/manage-address/:ucid', passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
+    const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid, false);
+
+    if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+        throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+    }
+
+    const data = {};
+    data.email = subscription.email;
+    data.cid = subscription.cid;
+    data.lcid = req.params.lcid;
+    data.title = list.name;
+    data.csrfToken = req.csrfToken();
+
+    data.template = {
+        template: 'subscription/web-manage-address.mjml.hbs',
+        layout: 'subscription/layout.mjml.hbs',
+        type: 'mjml'
+    };
+
+    await injectCustomFormData(req.query.fid || list.default_form, 'data/web-manage-address', data);
+
+    const htmlRenderer = await tools.getTemplate(data.template);
+
+    data.isWeb = true;
+    data.needsJsWarning = true;
+    data.isManagePreferences = true;
+    data.flashMessages = await captureFlashMessages(res);
+
+    res.send(htmlRenderer(data));
+});
+
+
+router.postAsync('/:lcid/manage-address', passport.parseForm, passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
+
+    const emailNew = cleanupFromPost(req.body['EMAIL_NEW']);
+
+    const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.body.cid, false);
+
+    if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+        throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+    }
+
+    if (subscription.email === emailNew) {
+        req.flash('info', tUI('subscription.nothingChanged', req.language));
+
+    } else {
+        const emailErr = await tools.validateEmail(emailNew);
+        if (emailErr) {
+            const errMsg = tools.validateEmailGetMessage(emailErr, email);
+
+            req.flash('danger', errMsg);
+
+        } else {
+            let newSubscription;
+            try {
+                newSubscription = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, emailNew, false);
+            } catch (err) {
+                if (err instanceof interoperableErrors.NotFoundError) {
+                } else {
+                    throw err;
+                }
+            }
+
+            if (newSubscription && newSubscription.status === SubscriptionStatus.SUBSCRIBED) {
+                await mailHelpers.sendAlreadySubscribed(req.language, list, emailNew, subscription);
+            } else {
+                const data = {
+                    subscriptionId: subscription.id,
+                    emailNew
+                };
+
+                const confirmCid = await confirmations.addConfirmation(list.id, 'change-address', req.ip, data);
+                await mailHelpers.sendConfirmAddressChange(req.language, list, emailNew, confirmCid, subscription);
+            }
+
+            req.flash('info', tUI('subscription.furtherInstructionsSent', req.language));
+        }
+    }
+
+    res.redirect('/subscription/' + encodeURIComponent(req.params.lcid) + '/manage/' + encodeURIComponent(req.body.cid));
+});
+
+
+router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['defaultAddress']);
+
+    const autoUnsubscribe = req.query.auto === 'yes';
+
+    if (autoUnsubscribe) {
+        await handleUnsubscribe(list, req.params.ucid, autoUnsubscribe, req.query.c, req.ip, req, res);
+
+    } else if (req.query.formTest ||
+        list.unsubscription_mode === lists.UnsubscriptionMode.ONE_STEP_WITH_FORM ||
+        list.unsubscription_mode === lists.UnsubscriptionMode.TWO_STEP_WITH_FORM) {
+
+        const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid, false);
+
+        if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+            throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+        }
+
+        const data = {};
+        data.email = subscription.email;
+        data.lcid = req.params.lcid;
+        data.ucid = req.params.ucid;
+        data.title = list.name;
+        data.csrfToken = req.csrfToken();
+        data.campaign = req.query.c;
+        data.defaultAddress = configItems.defaultAddress;
+
+        data.template = {
+            template: 'subscription/web-unsubscribe.mjml.hbs',
+            layout: 'subscription/layout.mjml.hbs',
+            type: 'mjml'
+        };
+
+        await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-unsubscribe', data);
+
+        const htmlRenderer = await tools.getTemplate(data.template);
+
+        data.isWeb = true;
+        data.needsJsWarning = true;
+        data.isManagePreferences = true;
+        data.flashMessages = await captureFlashMessages(res);
+
+        res.send(htmlRenderer(data));
+
+    } else { // UnsubscriptionMode.ONE_STEP || UnsubscriptionMode.TWO_STEP || UnsubscriptionMode.MANUAL
+        await handleUnsubscribe(list, req.params.ucid, autoUnsubscribe, req.query.c, req.ip, req, res);
+    }
+});
+
+
+router.postAsync('/:lcid/unsubscribe', passport.parseForm, passport.csrfProtection, async (req, res) => {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
+
+    const campaignCid = cleanupFromPost(req.body.campaign);
+
+    await handleUnsubscribe(list, req.body.ucid, false, campaignCid, req.ip, req, res);
+});
+
+
+async function handleUnsubscribe(list, subscriptionCid, autoUnsubscribe, campaignCid, ip, req, res) {
+    if ((list.unsubscription_mode === lists.UnsubscriptionMode.ONE_STEP || list.unsubscription_mode === lists.UnsubscriptionMode.ONE_STEP_WITH_FORM) ||
+        (autoUnsubscribe && (list.unsubscription_mode === lists.UnsubscriptionMode.TWO_STEP || list.unsubscription_mode === lists.UnsubscriptionMode.TWO_STEP_WITH_FORM)) ) {
+
+        try {
+            const subscription = await subscriptions.unsubscribeByCidAndGet(contextHelpers.getAdminContext(), list.id, subscriptionCid, campaignCid);
+
+            await mailHelpers.sendUnsubscriptionConfirmed(req.language, list, subscription.email, subscription);
+
+            res.redirect('/subscription/' + encodeURIComponent(list.cid) + '/unsubscribed-notice');
+
+        } catch (err) {
+            if (err instanceof interoperableErrors.NotFoundError) {
+                throw new interoperableErrors.NotFoundError('Subscription not found in this list'); // This is here to provide some meaningful error message.
+            }
+        }
+
+    } else {
+        const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid, false);
+
+        if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+            throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+        }
+
+        if (list.unsubscription_mode === lists.UnsubscriptionMode.TWO_STEP || list.unsubscription_mode === lists.UnsubscriptionMode.TWO_STEP_WITH_FORM) {
+
+            const data = {
+                subscriptionCid,
+                campaignCid
+            };
+
+            const confirmCid = await confirmations.addConfirmation(list.id, 'unsubscribe', ip, data);
+            await mailHelpers.sendConfirmUnsubscription(req.language, list, subscription.email, confirmCid, subscription);
+
+            res.redirect('/subscription/' + encodeURIComponent(list.cid) + '/confirm-unsubscription-notice');
+
+        } else { // UnsubscriptionMode.MANUAL
+            res.redirect('/subscription/' + encodeURIComponent(list.cid) + '/manual-unsubscribe-notice');
+        }
+    }
+}
+
+
+router.getAsync('/:cid/confirm-subscription-notice', async (req, res) => {
+    await webNotice('confirm-subscription', req, res);
+});
+
+router.getAsync('/:cid/confirm-unsubscription-notice', async (req, res) => {
+    await webNotice('confirm-unsubscription', req, res);
+});
+
+router.getAsync('/:cid/subscribed-notice', async (req, res) => {
+    await webNotice('subscribed', req, res);
+});
+
+router.getAsync('/:cid/updated-notice', async (req, res) => {
+    await webNotice('updated', req, res);
+});
+
+router.getAsync('/:cid/unsubscribed-notice', async (req, res) => {
+    await webNotice('unsubscribed', req, res);
+});
+
+router.getAsync('/:cid/manual-unsubscribe-notice', async (req, res) => {
+    await webNotice('manual-unsubscribe', req, res);
+});
+
+router.postAsync('/publickey', passport.parseForm, async (req, res) => {
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPassphrase', 'pgpPrivateKey']);
+
+    if (!configItems.pgpPrivateKey) {
+        const err = new Error('Public key is not set');
+        err.status = 404;
+        throw err;
+    }
+
+    let privKey;
+    try {
+        privKey = openpgp.key.readArmored(configItems.pgpPrivateKey).keys[0];
+        if (configItems.pgpPassphrase && !privKey.decrypt(configItems.pgpPassphrase)) {
+            privKey = false;
+        }
+    } catch (E) {
+        // just ignore if failed
+    }
+
+    if (!privKey) {
+        const err = new Error('Public key is not set');
+        err.status = 404;
+        throw err;
+    }
+
+    const pubkey = privKey.toPublic().armor();
+
+    res.writeHead(200, {
+        'Content-Type': 'application/octet-stream',
+        'Content-Disposition': 'attachment; filename=public.asc'
+    });
+
+    res.end(pubkey);
+});
+
+
+async function webNotice(type, req, res) {
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+
+    const configItems = await settings.get(contextHelpers.getAdminContext(), ['defaultHomepage', 'adminEmail']);
+
+    const data = {
+        title: list.name,
+        homepage: configItems.defaultHomepage || getTrustedUrl(),
+        contactAddress: list.from_email || configItems.adminEmail,
+        template: {
+            template: 'subscription/web-' + type + '-notice.mjml.hbs',
+            layout: 'subscription/layout.mjml.hbs',
+            type: 'mjml'
+        }
+    };
+
+    await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-' + type + '-notice', data);
+
+    const htmlRenderer = await tools.getTemplate(data.template);
+
+    data.isWeb = true;
+    data.isConfirmNotice = true; // FIXME: Not sure what this does. Check it in a browser with disabled JS
+    data.isManagePreferences = true;
+    data.flashMessages = await captureFlashMessages(res);
+
+    res.send(htmlRenderer(data));
+}
+
+module.exports = router;

server/routes/subscriptions.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const passport = require('../lib/passport');
+const shares = require('../models/shares');
+const contextHelpers = require('../lib/context-helpers');
+const router = require('../lib/router-async').create();
+const subscriptions = require('../models/subscriptions');
+const {castToInteger} = require('../lib/helpers');
+const stringify = require('csv-stringify')
+const fields = require('../models/fields');
+const lists = require('../models/lists');
+const moment = require('moment');
+
+router.getAsync('/export/:listId/:segmentId', passport.loggedIn, async (req, res) => {
+    const listId = castToInteger(req.params.listId);
+    const segmentId = castToInteger(req.params.segmentId);
+
+    const flds = await fields.list(req.context, listId);
+
+    const columns = [
+        {key: 'cid', header: 'cid'},
+        {key: 'hash_email', header: 'HASH_EMAIL'},
+        {key: 'email', header: 'EMAIL'},
+    ];
+
+    for (const fld of flds) {
+        if (fld.column) {
+            columns.push({
+                key: fld.column,
+                header: fld.key
+            });
+        }
+    }
+
+    const list = await lists.getById(req.context, listId);
+
+    const headers = {
+        'Content-Disposition': `attachment;filename=subscriptions-${list.cid}-${segmentId}-${moment().toISOString()}.csv`,
+        'Content-Type': 'text/csv'
+    };
+
+    res.set(headers);
+
+    const stringifier = stringify({
+        columns,
+        header: true,
+        delimiter: ','
+    });
+
+    stringifier.pipe(res);
+
+    for await (const subscription of subscriptions.listIterator(req.context, listId, segmentId, false)) {
+        stringifier.write(subscription);
+    }
+
+    stringifier.end();
+});
+
+module.exports = router;

server/routes/webhooks.js

@@ -0,0 +1,240 @@
+'use strict';
+
+const router = require('../lib/router-async').create();
+const request = require('request-promise');
+const campaigns = require('../models/campaigns');
+const sendConfigurations = require('../models/send-configurations');
+const contextHelpers = require('../lib/context-helpers');
+const {SubscriptionStatus} = require('../../shared/lists');
+const {MailerType} = require('../../shared/send-configurations');
+const log = require('../lib/log');
+const multer = require('multer');
+const uploads = multer();
+
+
+router.postAsync('/aws', async (req, res) => {
+    if (typeof req.body === 'string') {
+        req.body = JSON.parse(req.body);
+    }
+
+    switch (req.body.Type) {
+
+        case 'SubscriptionConfirmation':
+            if (req.body.SubscribeURL) {
+                await request(req.body.SubscribeURL);
+                break;
+            } else {
+                const err = new Error('SubscribeURL not set');
+                err.status = 400;
+                throw err;
+            }
+
+        case 'Notification':
+            if (req.body.Message) {
+                if (typeof req.body.Message === 'string') {
+                    req.body.Message = JSON.parse(req.body.Message);
+                }
+
+                if (req.body.Message.mail && req.body.Message.mail.messageId) {
+                    const message = await campaigns.getMessageByResponseId(req.body.Message.mail.messageId);
+
+                    if (!message) {
+                        return;
+                    }
+
+                    switch (req.body.Message.notificationType) {
+                        case 'Bounce':
+                            await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.BOUNCED, req.body.Message.bounce.bounceType === 'Permanent');
+                            log.verbose('AWS', 'Marked message %s as bounced', req.body.Message.mail.messageId);
+                            break;
+
+                        case 'Complaint':
+                            if (req.body.Message.complaint) {
+                                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.COMPLAINED, true);
+                                log.verbose('AWS', 'Marked message %s as complaint', req.body.Message.mail.messageId);
+                            }
+                            break;
+                    }
+                }
+            }
+            break;
+    }
+
+    res.json({
+        success: true
+    });
+});
+
+
+router.postAsync('/sparkpost', async (req, res) => {
+    const events = [].concat(req.body || []); // This is just a cryptic way getting an array regardless whether req.body is empty, one item, or array
+
+    for (const curEvent of events) {
+        let msys = curEvent && curEvent.msys;
+        let evt;
+
+        if (msys && msys.message_event) {
+            evt = msys.message_event;
+        } else if (msys && msys.unsubscribe_event) {
+            evt = msys.unsubscribe_event;
+        } else {
+            continue;
+        }
+
+        const message = await campaigns.getMessageByCid(evt.campaign_id);
+        if (!message) {
+            continue;
+        }
+
+        switch (evt.type) {
+            case 'bounce':
+                // https://support.sparkpost.com/customer/portal/articles/1929896
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.BOUNCED, [1, 10, 25, 30, 50].indexOf(Number(evt.bounce_class)) >= 0);
+                log.verbose('Sparkpost', 'Marked message %s as bounced', evt.campaign_id);
+                break;
+
+            case 'spam_complaint':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.COMPLAINED, true);
+                log.verbose('Sparkpost', 'Marked message %s as complaint', evt.campaign_id);
+                break;
+
+            case 'link_unsubscribe':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.UNSUBSCRIBED, true);
+                log.verbose('Sparkpost', 'Marked message %s as unsubscribed', evt.campaign_id);
+                break;
+        }
+    }
+
+    return res.json({
+        success: true
+    });
+});
+
+
+router.postAsync('/sendgrid', async (req, res) => {
+    let events = [].concat(req.body || []);
+
+    for (const evt of events) {
+        if (!evt) {
+            continue;
+        }
+
+        const message = await campaigns.getMessageByCid(evt.campaign_id);
+        if (!message) {
+            continue;
+        }
+
+        switch (evt.event) {
+            case 'bounce':
+                // https://support.sparkpost.com/customer/portal/articles/1929896
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.BOUNCED, true);
+                log.verbose('Sendgrid', 'Marked message %s as bounced', evt.campaign_id);
+                break;
+
+            case 'spamreport':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.COMPLAINED, true);
+                log.verbose('Sendgrid', 'Marked message %s as complaint', evt.campaign_id);
+                break;
+
+            case 'group_unsubscribe':
+            case 'unsubscribe':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.UNSUBSCRIBED, true);
+                log.verbose('Sendgrid', 'Marked message %s as unsubscribed', evt.campaign_id);
+                break;
+        }
+    }
+
+    return res.json({
+        success: true
+    });
+});
+
+
+router.postAsync('/mailgun', uploads.any(), async (req, res) => {
+    const evt = req.body;
+
+    const message = await campaigns.getMessageByCid([].concat(evt && evt.campaign_id || []).shift());
+    if (message) {
+        switch (evt.event) {
+            case 'bounced':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.BOUNCED, true);
+                log.verbose('Mailgun', 'Marked message %s as bounced', evt.campaign_id);
+                break;
+
+            case 'complained':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.COMPLAINED, true);
+                log.verbose('Mailgun', 'Marked message %s as complaint', evt.campaign_id);
+                break;
+
+            case 'unsubscribed':
+                await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.UNSUBSCRIBED, true);
+                log.verbose('Mailgun', 'Marked message %s as unsubscribed', evt.campaign_id);
+                break;
+        }
+    }
+
+    return res.json({
+        success: true
+    });
+});
+
+
+router.postAsync('/zone-mta', async (req, res) => {
+    if (typeof req.body === 'string') {
+        req.body = JSON.parse(req.body);
+    }
+
+    if (req.body.id) {
+        const message = await campaigns.getMessageByCid(req.body.id);
+
+        if (message) {
+            await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.BOUNCED, true);
+            log.verbose('ZoneMTA', 'Marked message %s as bounced', req.body.id);
+        }
+    }
+
+    res.json({
+        success: true
+    });
+});
+
+
+router.postAsync('/zone-mta/sender-config/:sendConfigurationCid', async (req, res) => {
+    if (!req.query.api_token) {
+        return res.json({
+            error: 'api_token value not set'
+        });
+    }
+
+    const sendConfiguration = await sendConfigurations.getByCid(contextHelpers.getAdminContext(), req.params.sendConfigurationCid, false, true);
+
+    if (sendConfiguration.mailer_type !== MailerType.ZONE_MTA || sendConfiguration.mailer_settings.dkimApiKey !== req.query.api_token) {
+        return res.json({
+            error: 'invalid api_token value'
+        });
+    }
+
+    const dkimDomain = sendConfiguration.mailer_settings.dkimDomain;
+    const dkimSelector = (sendConfiguration.mailer_settings.dkimSelector || '').trim();
+    const dkimPrivateKey = (sendConfiguration.mailer_settings.dkimPrivateKey || '').trim();
+
+    if (!dkimSelector || !dkimPrivateKey) {
+        // empty response
+        return res.json({});
+    }
+
+    const from = (req.body.from || '').trim();
+    const domain = from.split('@').pop().toLowerCase().trim();
+
+    res.json({
+        dkim: {
+            keys: [{
+                domainName: dkimDomain || domain,
+                keySelector: dkimSelector,
+                privateKey: dkimPrivateKey
+            }]
+        }
+    });
+});
+
+module.exports = router;

server/services/executor.js

@@ -0,0 +1,132 @@
+'use strict';
+
+/* Privileged executor. If Mailtrain is started as root, this process keeps the root privilege to be able to spawn workers
+   that can chroot.
+  */
+
+const reportHelpers = require('../lib/report-helpers');
+const fork = require('child_process').fork;
+const path = require('path');
+const log = require('../lib/log');
+const fs = require('fs');
+const privilegeHelpers = require('../lib/privilege-helpers');
+
+let processes = {};
+
+function spawnProcess(tid, executable, args, outFile, errFile, cwd, uid, gid) {
+
+    function reportFail(msg) {
+        process.send({
+            type: 'process-failed',
+            msg,
+            tid
+        });
+    }
+
+    fs.open(outFile, 'w', (err, outFd) => {
+        if (err) {
+            log.error('Executor', err);
+            reportFail('Cannot create standard output file');
+            return;
+        }
+
+        fs.open(errFile, 'w', (err, errFd) => {
+            if (err) {
+                log.error('Executor', err);
+                reportFail('Cannot create standard error file');
+                return;
+            }
+
+            privilegeHelpers.ensureMailtrainOwner(outFile, err => {
+                if (err) {
+                    log.warn('Executor', 'Cannot change owner of output file of process tid:%s', tid);
+                }
+
+                privilegeHelpers.ensureMailtrainOwner(errFile, err => {
+                    if (err) {
+                        log.warn('Executor', 'Cannot change owner of error output file of process tid:%s', tid);
+                    }
+
+                    const options = {
+                        stdio: ['ignore', outFd, errFd, 'ipc'],
+                        cwd,
+                        env: {NODE_ENV: process.env.NODE_ENV},
+                        uid,
+                        gid
+                    };
+
+                    let child;
+
+                    try {
+                        child = fork(executable, args, options);
+                    } catch (err) {
+                        log.error('Executor', 'Cannot start process with tid:%s', tid);
+                        reportFail('Cannot start process');
+                        return;
+                    }
+
+                    const pid = child.pid;
+                    processes[tid] = child;
+
+                    log.info('Executor', 'Process started with tid:%s pid:%s', tid, pid);
+                    process.send({
+                        type: 'process-started',
+                        tid
+                    });
+
+                    child.on('close', (code, signal) => {
+
+                        delete processes[tid];
+                        log.info('Executor', 'Process tid:%s pid:%s exited with code %s signal %s', tid, pid, code, signal);
+
+                        fs.close(outFd, err => {
+                            if (err) {
+                                log.error('Executor', err);
+                            }
+
+                            fs.close(errFd, err => {
+                                if (err) {
+                                    log.error('Executor', err);
+                                }
+
+                                process.send({
+                                    type: 'process-finished',
+                                    tid,
+                                    code,
+                                    signal
+                                });
+                            });
+                        });
+                    });
+                });
+            });
+        });
+    });
+}
+
+process.on('message', msg => {
+    if (msg) {
+        const type = msg.type;
+
+        if (type === 'start-report-processor-worker') {
+
+            const ids = privilegeHelpers.getConfigROUidGid();
+            spawnProcess(msg.tid, path.join(__dirname, '..', 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, '..', 'workers', 'reports'), ids.uid, ids.gid);
+
+        } else if (type === 'stop-process') {
+            const child = processes[msg.tid];
+
+            if (child) {
+                log.info('Executor', 'Killing process tid:%s pid:%s', msg.tid, child.pid);
+                child.kill();
+            } else {
+                log.info('Executor', 'No running process found with tid:%s pid:%s', msg.tid, child.pid);
+            }
+        }
+    }
+});
+
+process.send({
+    type: 'executor-started'
+});
+

server/services/feedcheck.js

@@ -0,0 +1,165 @@
+'use strict';
+
+const log = require('../lib/log');
+const knex = require('../lib/knex');
+const feedparser = require('feedparser-promised');
+const { CampaignType, CampaignStatus, CampaignSource } = require('../../shared/campaigns');
+const util = require('util');
+const campaigns = require('../models/campaigns');
+const contextHelpers = require('../lib/context-helpers');
+
+const { tLog } = require('../lib/translate');
+
+const feedCheckInterval = 10 * 60 * 1000;
+
+const dbCheckInterval = 60 * 1000;
+
+let running = false;
+
+async function fetch(url) {
+    const httpOptions = {
+        uri: 'http://feeds.feedwrench.com/JavaScriptJabber.rss',
+        headers: {
+            'user-agent': 'Mailtrain',
+            'accept': 'text/html,application/xhtml+xml'
+        }
+    };
+
+    const items = await feedparser.parse(httpOptions);
+
+    const entries = [];
+    for (const item of items) {
+        const entry = {
+            title: item.title,
+            date: item.date || item.pubdate || item.pubDate || new Date(),
+            guid: item.guid || item.link,
+            link: item.link,
+            content: item.description || item.summary,
+            summary: item.summary || item.description,
+            image_url: item.image.url
+        };
+        entries.push(entry);
+    }
+
+    return entries;
+}
+
+async function run() {
+    if (running) {
+        return;
+    }
+
+    running = true;
+
+    let rssCampaignIdRow;
+
+    while (rssCampaignIdRow = await knex('campaigns')
+        .where('type', CampaignType.RSS)
+        .where('status', CampaignStatus.ACTIVE)
+        .where(qry => qry.whereNull('last_check').orWhere('last_check', '<', new Date(Date.now() - feedCheckInterval)))
+        .select('id')
+        .first()) {
+
+        const rssCampaign = await campaigns.getById(contextHelpers.getAdminContext(), rssCampaignIdRow.id, false);
+
+        let checkStatus = null;
+
+        try {
+            const entries = await fetch(rssCampaign.data.feedUrl);
+
+            let added = 0;
+
+            for (const entry of entries) {
+                let entryId = null;
+
+                await knex.transaction(async tx => {
+                    const existingEntry = await tx('rss').where({
+                        parent: rssCampaign.id,
+                        guid: entry.guid
+                    }).first();
+
+                    if (!existingEntry) {
+                        const campaignData = {};
+
+                        let source = rssCampaign.source;
+                        if (source === CampaignSource.CUSTOM_FROM_TEMPLATE || source === CampaignSource.CUSTOM) {
+                            source = CampaignSource.CUSTOM_FROM_CAMPAIGN;
+                            campaignData.sourceCampaign = rssCampaign.id;
+                        } else {
+                            Object.assign(campaignData, rssCampaign.data);
+                        }
+
+                        campaignData.rssEntry = entry;
+
+                        const campaign = {
+                            parent: rssCampaign.id,
+                            type: CampaignType.RSS_ENTRY,
+                            source,
+                            name: entry.title || `RSS entry ${entry.guid.substr(0, 67)}`,
+                            lists: rssCampaign.lists,
+                            namespace: rssCampaign.namespace,
+                            send_configuration: rssCampaign.send_configuration,
+
+                            from_name_override: rssCampaign.from_name_override,
+                            from_email_override: rssCampaign.from_email_override,
+                            reply_to_override: rssCampaign.reply_to_override,
+                            subject_override: rssCampaign.subject_override,
+                            data: campaignData,
+
+                            click_tracking_disabled: rssCampaign.click_tracking_disabled,
+                            open_tracking_disabled: rssCampaign.open_tracking_disabled,
+                            unsubscribe_url: rssCampaign.unsubscribe_url
+                        };
+
+                        const ids = await campaigns.createRssTx(tx, contextHelpers.getAdminContext(), campaign);
+                        const campaignId = ids[0];
+
+                        await tx('rss').insert({
+                            parent: rssCampaign.id,
+                            campaign: campaignId,
+                            guid: entry.guid,
+                            pubdate: entry.date,
+                        });
+
+                        added += 1;
+                    }
+                });
+            }
+
+            if (added > 0) {
+                checkStatus = tLog('feedCheck.campaignsAdded', {addedMessages: added, campaignId: rssCampaign.id});
+                log.verbose('Feed', `Found ${added} new campaigns messages from feed ${rssCampaign.id}`);
+
+                process.send({
+                    type: 'entries-added'
+                });
+            } else {
+                checkStatus = tLog('feedCheck.nothingNew');
+            }
+
+            rssCampaign.data.checkStatus = checkStatus;
+            await knex('campaigns').where('id', rssCampaign.id).update({
+                last_check: Date.now(),
+                data: JSON.stringify(rssCampaign.data)
+            });
+
+        } catch (err) {
+            log.error('Feed', err.message);
+            rssCampaign.data.checkStatus = err.message;
+            await knex('campaigns').where('id', rssCampaign.id).update({
+                last_check: Date.now(),
+                data: JSON.stringify(rssCampaign.data)
+            });
+        }
+    }
+
+    running = false;
+
+    setTimeout(run, dbCheckInterval);
+}
+
+process.send({
+    type: 'feedcheck-started'
+});
+
+run();

server/services/importer.js

@@ -0,0 +1,410 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const path = require('path');
+const log = require('../lib/log');
+const fsExtra = require('fs-extra-promise');
+const {ImportSource, MappingType, ImportStatus, RunStatus} = require('../../shared/imports');
+const imports = require('../models/imports');
+const fields = require('../models/fields');
+const subscriptions = require('../models/subscriptions');
+const { Writable } = require('stream');
+const { cleanupFromPost, enforce } = require('../lib/helpers');
+const contextHelpers = require('../lib/context-helpers');
+const tools = require('../lib/tools');
+const shares = require('../models/shares');
+const { tLog } = require('../lib/translate');
+
+
+const csvparse = require('csv-parse');
+const fs = require('fs');
+
+let running = false;
+const maxPrepareBatchSize = 100;
+const maxImportBatchSize = 10;
+
+function prepareCsv(impt) {
+    // Processing of CSV intake
+    const filePath = path.join(imports.filesDir, impt.settings.csv.filename);
+    const importTable = 'import_file__' + impt.id;
+
+    let finishedWithError = false;
+    let firstRow;
+
+    const finishWithError = async (msg, err) => {
+        finishedWithError = true;
+        log.error('Importer (CSV)', err.stack);
+
+        await knex('imports').where('id', impt.id).update({
+            status: ImportStatus.PREP_FAILED,
+            error: msg + '\n' + err.message
+        });
+
+        await fsExtra.removeAsync(filePath);
+    };
+
+    const finishWithSuccess = async () => {
+        if (finishedWithError) {
+            return;
+        }
+
+        log.info('Importer (CSV)', 'Preparation finished');
+
+        await knex('imports').where('id', impt.id).update({
+            status: ImportStatus.PREP_FINISHED,
+            error: null
+        });
+
+        await fsExtra.removeAsync(filePath);
+    };
+
+    const processRows = async (chunks) => {
+        let insertBatch = [];
+        for (const chunkEntry of chunks) {
+            const record = chunkEntry.chunk;
+
+            if (!firstRow) {
+                firstRow = true;
+
+                const cols = [];
+                let colsDef = '';
+                for (let idx = 0; idx < record.length; idx++) {
+                    const colName = 'column_' + idx;
+                    cols.push({
+                        column: colName,
+                        name: record[idx]
+                    });
+
+                    colsDef += '  `' + colName + '` text DEFAULT NULL,\n';
+                }
+
+                impt.settings.csv.columns = cols;
+                impt.settings.sourceTable = importTable;
+
+                await knex('imports').where({id: impt.id}).update({settings: JSON.stringify(impt.settings)});
+
+                await knex.schema.raw('CREATE TABLE `' + importTable + '` (\n' +
+                    '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
+                    colsDef +
+                    '  PRIMARY KEY (`id`)\n' +
+                    ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+            } else {
+                const dbRecord = {};
+                for (let idx = 0; idx < record.length; idx++) {
+                    dbRecord['column_' + idx] = record[idx];
+                }
+
+                insertBatch.push(dbRecord);
+            }
+
+            if (insertBatch.length >= maxPrepareBatchSize) {
+                await knex(importTable).insert(insertBatch);
+                insertBatch = [];
+            }
+        }
+
+        if (insertBatch.length > 0) {
+            await knex(importTable).insert(insertBatch);
+        }
+    };
+
+
+    const inputStream = fs.createReadStream(filePath);
+    const parser = csvparse({
+        comment: '#',
+        delimiter: impt.settings.csv.delimiter
+    });
+
+    inputStream.on('error', err => finishWithError('Error reading CSV file.', err));
+    parser.on('error', err => finishWithError('Error parsing CSV file.', err));
+
+    const importProcessor = new Writable({
+        write(chunk, encoding, callback) {
+            processRows([{chunk, encoding}]).then(() => callback());
+        },
+        writev(chunks, callback) {
+            processRows(chunks).then(() => callback());
+        },
+        final(callback) {
+            finishWithSuccess().then(() => callback());
+        },
+        objectMode: true
+    });
+
+    parser.pipe(importProcessor);
+    inputStream.pipe(parser);
+}
+
+async function _execImportRun(impt, handlers) {
+    try {
+        let imptRun;
+
+        // It should not really happen that we have more than one run to be processed for an import. However, to be on the safe side, we process it in a while.
+        while (imptRun = await knex('import_runs').where('import', impt.id).whereIn('status', [RunStatus.SCHEDULED]).orderBy('created', 'asc').first()) {
+            try {
+                imptRun.mapping = JSON.parse(imptRun.mapping) || {};
+
+                log.info('Importer', `Starting BASIC_SUBSCRIBE run ${impt.id}.${imptRun.id}`);
+                await knex('import_runs').where('id', imptRun.id).update({
+                    status: RunStatus.RUNNING
+                });
+
+                const importTable = impt.settings.sourceTable;
+
+                const flds = await fields.list(contextHelpers.getAdminContext(), impt.list);
+
+                let lastId = imptRun.last_id || 0;
+                let countNew = imptRun.new || 0;
+                let countProcessed = imptRun.processed || 0;
+                let countFailed = imptRun.failed || 0;
+
+                while (true) {
+                    const rows = await knex(importTable).orderBy('id', 'asc').where('id', '>', lastId).limit(maxImportBatchSize);
+                    log.verbose('Importer', `Processing run ${impt.id}.${imptRun.id} with id > ${lastId} ... ${rows.length} entries`);
+
+                    if (rows.length === 0) {
+                        break;
+                    }
+
+                    const subscrs = [];
+                    const unsubscrs = [];
+                    const failures = [];
+
+                    // This should help in case we do the DNS check inside process row because it does all the checks at the same time.
+                    await Promise.all(rows.map(row => handlers.processSourceRow(impt, imptRun, flds, row, subscrs, unsubscrs, failures)));
+
+                    lastId = rows[rows.length - 1].id;
+
+                    await knex.transaction(async tx => {
+                        const groupedFieldsMap = await subscriptions.getGroupedFieldsMapTx(tx, impt.list);
+
+                        let newRows = 0;
+
+                        for (const subscr of subscrs) {
+                            const meta = {
+                                updateAllowed: true,
+                                updateOfUnsubscribedAllowed: true,
+                                subscribeIfNoExisting: true
+                            };
+
+                            try {
+                                await subscriptions.createTxWithGroupedFieldsMap(tx, contextHelpers.getAdminContext(), impt.list, groupedFieldsMap, subscr, impt.id, meta);
+                                if (!meta.existing) {
+                                    newRows += 1;
+                                }
+
+                            } catch (err) {
+                                failures.push({
+                                    run: imptRun.id,
+                                    source_id: subscr.source_id,
+                                    email: subscr.email,
+                                    reason: err.message
+                                });
+                            }
+                        }
+
+                        for (const unsubscr of unsubscrs) {
+                            try {
+                                await subscriptions.unsubscribeByEmailAndGetTx(tx, contextHelpers.getAdminContext(), impt.list, unsubscr.email);
+                            } catch (err) {
+                                failures.push({
+                                    run: imptRun.id,
+                                    source_id: unsubscr.source_id,
+                                    email: unsubscr.email,
+                                    reason: err.message
+                                });
+                            }
+                        }
+
+                        countProcessed += rows.length;
+                        countNew += newRows;
+                        countFailed += failures.length;
+
+                        if (failures.length > 0) {
+                            await tx('import_failed').insert(failures);
+                        }
+                        await tx('import_runs').where('id', imptRun.id).update({
+                            last_id: lastId,
+                            new: countNew,
+                            failed: countFailed,
+                            processed: countProcessed
+                        });
+                    });
+
+                    const imptRunStatus = await knex('import_runs').where('id', imptRun.id).select(['status']).first();
+                    if (imptRunStatus.status === RunStatus.STOPPING) {
+                        throw new Error('Aborted');
+                    }
+                }
+
+                await knex('import_runs').where('id', imptRun.id).update({
+                    status: RunStatus.FINISHED,
+                    error: null,
+                    finished: new Date()
+                });
+
+                log.info('Importer', `BASIC_SUBSCRIBE run ${impt.id}.${imptRun.id} finished`);
+            } catch (err) {
+                await knex('import_runs').where('id', imptRun.id).update({
+                    status: RunStatus.FAILED,
+                    error: err.message,
+                    finished: new Date()
+                });
+
+                throw new Error('Last run failed');
+            }
+        }
+
+        await knex('imports').where('id', impt.id).update({
+            last_run: new Date(),
+            error: null,
+            status: ImportStatus.RUN_FINISHED
+        });
+
+    } catch (err) {
+        await knex('imports').where('id', impt.id).update({
+            last_run: new Date(),
+            error: err.message,
+            status: ImportStatus.RUN_FAILED
+        });
+    }
+}
+
+async function basicSubscribe(impt) {
+    const handlers = {
+        processSourceRow: async (impt, imptRun, flds, row, subscriptions, unsubscriptions, failures) => {
+            const mappingFields = imptRun.mapping.fields || {};
+            const mappingSettings = imptRun.mapping.settings || {};
+
+            const convRow = {};
+            for (const col in mappingFields) {
+                const fldMapping = mappingFields[col];
+
+                if (fldMapping && fldMapping.column) {
+                    convRow[col] = row[fldMapping.column];
+                }
+            }
+
+            const subscription = fields.fromImport(impt.list, flds, convRow);
+            const email = cleanupFromPost(convRow.email);
+
+            let errorMsg;
+
+            if (!email) {
+                errorMsg = tLog('importer.missingEmail');
+            }
+
+            if (mappingSettings.checkEmails) {
+                const emailErr = await tools.validateEmail(email);
+                if (emailErr) {
+                    errorMsg = tools.validateEmailGetMessage(emailErr, email);
+                }
+            }
+
+            if (!errorMsg) {
+                subscription.email = email;
+                subscription.source_id = row.id;
+                subscriptions.push(subscription);
+            } else {
+                failures.push({
+                    run: imptRun.id,
+                    source_id: row.id,
+                    email: email,
+                    reason: errorMsg
+                });
+            }
+        }
+    };
+
+    return await _execImportRun(impt, handlers);
+}
+
+async function basicUnsubscribe(impt) {
+    const handlers = {
+        processSourceRow: async (impt, imptRun, flds, row, subscriptions, unsubscriptions, failures) => {
+            const emailCol = imptRun.mapping.fields.email.column;
+            const email = cleanupFromPost(row[emailCol]);
+
+            let errorMsg;
+
+            if (!email) {
+                errorMsg = tLog('importer.missingEmail');
+            }
+
+            if (!errorMsg) {
+                unsubscriptions.push({
+                    source_id: row.id,
+                    email
+                });
+            } else {
+                failures.push({
+                    run: imptRun.id,
+                    source_id: row.id,
+                    email: email,
+                    reason: errorMsg
+                });
+            }
+        }
+    };
+
+    return await _execImportRun(impt, handlers);
+}
+
+async function getTask() {
+    return await knex.transaction(async tx => {
+        const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first();
+
+        if (impt) {
+            impt.settings = JSON.parse(impt.settings) || {};
+
+            if (impt.source === ImportSource.CSV_FILE && impt.status === ImportStatus.PREP_SCHEDULED) {
+                await tx('imports').where('id', impt.id).update('status', ImportStatus.PREP_RUNNING);
+                return () => prepareCsv(impt);
+
+            } else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.mapping_type === MappingType.BASIC_SUBSCRIBE) {
+                await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
+                return () => basicSubscribe(impt);
+
+            } else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.mapping_type === MappingType.BASIC_UNSUBSCRIBE) {
+                await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
+                return () => basicUnsubscribe(impt);
+            }
+
+        } else {
+            return null;
+        }
+    });
+}
+
+async function run() {
+    if (running) {
+        return;
+    }
+
+    running = true;
+
+    let task;
+    while ((task = await getTask()) != null) {
+        task();
+    }
+
+    running = false;
+}
+
+process.on('message', msg => {
+    if (msg) {
+        const type = msg.type;
+
+        if (type === 'scheduleCheck') {
+            run();
+        }
+    }
+});
+
+process.send({
+    type: 'importer-started'
+});
+
+run();
+

server/services/postfix-bounce-server.js

@@ -0,0 +1,125 @@
+'use strict';
+
+const log = require('../lib/log');
+const config = require('config');
+const net = require('net');
+const campaigns = require('../models/campaigns');
+const contextHelpers = require('../lib/context-helpers');
+const { SubscriptionStatus } = require('../../shared/lists');
+
+const seenIds = new Set();
+
+let remainder = '';
+let reading = false;
+
+async function readNextChunks() {
+    if (reading) {
+        return false;
+    }
+
+    reading = true;
+
+    while (true) {
+        const chunk = socket.read();
+        if (chunk === null) {
+            reading = false;
+            return;
+        }
+
+        const lines = (remainder + chunk.toString()).split(/\r?\n/);
+        remainder = lines.pop();
+
+        for (const line of lines) {
+            try {
+                const match = /\bstatus=(bounced|sent)\b/.test(line) && line.match(/\bpostfix\/\w+\[\d+\]:\s*([^:]+).*?status=(\w+)/);
+                if (match) {
+                    let queueId = match[1];
+                    let queued = '';
+                    let queuedAs = '';
+
+                    if (!seenIds.has(queueId)) {
+                        seenIds.add(queueId);
+
+                        // Losacno: Check for local requeue
+                        let status = match[2];
+                        log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status);
+                        if (status === 'sent') {
+                            // Save new queueId to update message's previous queueId (thanks @mfechner )
+                            queued = / relay=/.test(line) && line.match(/status=sent \((.*)\)/);
+                            if (queued) {
+                                queued = queued[1];
+                                queuedAs = queued.match(/ queued as (\w+)/);
+                                if (queuedAs) {
+                                    queuedAs = queuedAs[1];
+                                } else {
+                                    queuedAs = '';
+                                }
+                            }
+                        }
+
+                        const message = await campaigns.getMessageByResponseId(queueId);
+                        if (message) {
+                            if (queuedAs || status === 'sent') {
+                                log.verbose('POSTFIXBOUNCE', 'Message %s locally requeued as %s', queueId, queuedAs);
+                                // Update message's previous queueId (thanks @mfechner )
+                                campaigns.updateMessageResponse(contextHelpers.getAdminContext(), message, queued, queuedAs);
+                                log.verbose('POSTFIXBOUNCE', 'Successfully changed message queueId to %s', queuedAs);
+                            } else {
+                                campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), message, SubscriptionStatus.BOUNCED, true);
+                                log.verbose('POSTFIXBOUNCE', 'Marked message %s as bounced', queueId);
+                            }
+
+                            // No need to keep in memory... free it ( thanks @witzig )
+                            seenIds.delete(queueId);
+                        }
+                    }
+                }
+            } catch (err) {
+                log.error('POSTFIXBOUNCE', err && err.stack);
+            }
+        }
+
+    }
+}
+
+module.exports = callback => {
+    if (!config.postfixbounce.enabled) {
+        return setImmediate(callback);
+    }
+
+    let started = false; // Not sure why all this magic around "started". But it was there this way in Mailtrain v1, so we kept it.
+
+    const server = net.createServer(socket => {
+        socket.on('readable', readNextChunks);
+    });
+
+    server.on('error', err => {
+        const port = config.postfixbounce.port;
+        const bind = typeof port === 'string' ? 'Pipe ' + port : 'Port ' + port;
+
+        switch (err.code) {
+            case 'EACCES':
+                log.error('POSTFIXBOUNCE', '%s requires elevated privileges.', bind);
+                break;
+            case 'EADDRINUSE':
+                log.error('POSTFIXBOUNCE', '%s is already in use', bind);
+                break;
+            default:
+                log.error('POSTFIXBOUNCE', err);
+        }
+
+        if (!started) {
+            started = true;
+            return callback(err);
+        }
+    });
+
+    server.listen(config.postfixbounce.port, config.postfixbounce.host, () => {
+        if (started) {
+            return server.close();
+        }
+        started = true;
+        log.info('POSTFIXBOUNCE', 'Server listening on port %s', config.postfixbounce.port);
+        setImmediate(callback);
+    });
+};