Work on sending campaigns. Campaign status page half-way done, but does not work yet.

client/src/account/API.js

@@ -31,6 +31,7 @@ export default class API extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.loadAccessToken();
     }
 

client/src/account/Account.js

@@ -42,6 +42,7 @@ export default class Account extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.loadFormValues();
     }
 

client/src/account/Reset.js

@@ -60,6 +60,7 @@ export default class Account extends Component {
             password2: ''
         });
 
+        // noinspection JSIgnoredPromiseFromCall
         this.validateResetToken();
     }
 

client/src/campaigns/CUD.js

@@ -45,12 +45,13 @@ import {getUrl} from "../lib/urls";
 import {
     campaignOverridables,
     CampaignSource,
+    CampaignStatus,
     CampaignType
 } from "../../../shared/campaigns";
 import moment from 'moment';
 import {getMailerTypes} from "../send-configurations/helpers";
 import {ResourceType} from "../lib/mosaico";
-import {getCampaignTypeLabels} from "./helpers";
+import {getCampaignLabels} from "./helpers";
 
 @translate()
 @withForm
@@ -66,7 +67,8 @@ export default class CUD extends Component {
         this.templateTypes = getTemplateTypes(props.t, 'data_sourceCustom_', ResourceType.CAMPAIGN);
         this.mailerTypes = getMailerTypes(props.t);
 
-        this.campaignTypes = getCampaignTypeLabels(t);
+        const { campaignTypeLabels } = getCampaignLabels(t);
+        this.campaignTypeLabels = campaignTypeLabels;
 
         this.createTitles = {
             [CampaignType.REGULAR]: t('Create Regular Campaign'),
@@ -134,6 +136,7 @@ export default class CUD extends Component {
 
     onSendConfigurationChanged(newState, key, oldValue, sendConfigurationId) {
         newState.sendConfiguration = null;
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchSendConfiguration(sendConfigurationId);
     }
 
@@ -167,7 +170,7 @@ export default class CUD extends Component {
                 }
 
                 for (const overridable of campaignOverridables) {
-                    data[overridable + '_overriden'] = !!data[overridable + '_override'];
+                    data[overridable + '_overriden'] = data[overridable + '_override'] === null;
                 }
 
                 const lsts = [];
@@ -184,9 +187,14 @@ export default class CUD extends Component {
                 }
                 data.lists = lsts;
 
+                // noinspection JSIgnoredPromiseFromCall
                 this.fetchSendConfiguration(data.send_configuration);
             });
 
+            if (this.props.entity.status === CampaignStatus.SENDING) {
+                this.disableForm();
+            }
+
         } else {
             const data = {};
             for (const overridable of campaignOverridables) {
@@ -620,7 +628,7 @@ export default class CUD extends Component {
             const campaignsColumns = [
                 { data: 1, title: t('Name') },
                 { data: 2, title: t('Description') },
-                { data: 3, title: t('Type'), render: data => this.campaignTypes[data] },
+                { data: 3, title: t('Type'), render: data => this.campaignTypeLabels[data] },
                 { data: 4, title: t('Created'), render: data => moment(data).fromNow() },
                 { data: 5, title: t('Namespace') }
             ];
@@ -671,6 +679,12 @@ export default class CUD extends Component {
 
                 <Title>{isEdit ? this.editTitles[this.getFormValue('type')] : this.createTitles[this.getFormValue('type')]}</Title>
 
+                {isEdit && this.props.entity.status === CampaignStatus.SENDING &&
+                    <div className={`alert alert-info`} role="alert">
+                        {t('Form cannot be edited because the campaign is currently being sent out. Wait till the sending is finished and refresh.')}
+                    </div>
+                }
+
                 <Form stateOwner={this} onSubmitAsync={::this.submitHandler}>
                     <InputField id="name" label={t('Name')}/>
                     <TextArea id="description" label={t('Description')}/>

client/src/campaigns/List.js

@@ -25,7 +25,7 @@ import {
     CampaignType
 } from "../../../shared/campaigns";
 import {checkPermissions} from "../lib/permissions";
-import {getCampaignTypeLabels} from "./helpers";
+import {getCampaignLabels} from "./helpers";
 
 @translate()
 @withPageHelpers
@@ -37,15 +37,9 @@ export default class List extends Component {
 
         const t = props.t;
 
-        this.campaignStatuses = {
+        const { campaignTypeLabels, campaignStatusLabels } = getCampaignLabels(t);
-            [CampaignStatus.IDLE]: t('Idle'),
+        this.campaignTypeLabels = campaignTypeLabels;
-            [CampaignStatus.FINISHED]: t('Finished'),
+        this.campaignStatusLabels = campaignStatusLabels;
-            [CampaignStatus.PAUSED]: t('Paused'),
-            [CampaignStatus.INACTIVE]: t('Inactive'),
-            [CampaignStatus.ACTIVE]: t('Active')
-        };
-
-        this.campaignTypes = getCampaignTypeLabels(t);
 
         this.state = {};
     }
@@ -65,6 +59,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 
@@ -74,7 +69,7 @@ export default class List extends Component {
         const columns = [
             { data: 1, title: t('Name') },
             { data: 2, title: t('Description') },
-            { data: 3, title: t('Type'), render: data => this.campaignTypes[data] },
+            { data: 3, title: t('Type'), render: data => this.campaignTypeLabels[data] },
             {
                 data: 4,
                 title: t('Status'),
@@ -87,7 +82,7 @@ export default class List extends Component {
                             return t('Sending');
                         }
                     } else {
-                        return this.campaignStatuses[data];
+                        return this.campaignStatusLabels[data];
                     }
                 }
             },
@@ -100,6 +95,13 @@ export default class List extends Component {
                     const campaignType = data[3];
                     const campaignSource = data[6];
 
+                    if (perms.includes('viewStats')) {
+                        actions.push({
+                            label: <Icon icon="send" title={t('Status')}/>,
+                            link: `/campaigns/${data[0]}/status`
+                        });
+                    }
+
                     if (perms.includes('edit')) {
                         actions.push({
                             label: <Icon icon="edit" title={t('Edit')}/>,

client/src/campaigns/Status.js

@@ -0,0 +1,387 @@
+'use strict';
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import {translate} from 'react-i18next';
+import {
+    requiresAuthenticatedUser,
+    Title,
+    withPageHelpers
+} from '../lib/page';
+import {
+    AlignedRow,
+    ButtonRow,
+    CheckBox,
+    DatePicker,
+    Form,
+    InputField,
+    TableSelect,
+    withForm
+} from '../lib/form';
+import {
+    withAsyncErrorHandler,
+    withErrorHandling
+} from '../lib/error-handling';
+import {getCampaignLabels} from './helpers';
+import {Table} from "../lib/table";
+import {Button} from "../lib/bootstrap-components";
+import axios from "../lib/axios";
+import {getUrl} from "../lib/urls";
+import interoperableErrors from '../../../shared/interoperable-errors';
+import {CampaignStatus} from "../../../shared/campaigns";
+import moment from 'moment';
+import campaignsStyles from "./styles.scss";
+
+
+@translate()
+@withForm
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+class TestUser extends Component {
+    constructor(props) {
+        super(props);
+        this.initForm();
+    }
+
+    static propTypes = {
+        entity: PropTypes.object.isRequired
+    }
+
+    localValidateFormValues(state) {
+        const t = this.props.t;
+
+        if (!state.getIn(['testUser', 'value'])) {
+            state.setIn(['testUser', 'error'], t('Subscription has to be selected to show the campaign for a test user.'))
+        } else {
+            state.setIn(['testUser', 'error'], null);
+        }
+    }
+
+    componentDidMount() {
+        this.populateFormValues({
+            testUser: null,
+        });
+    }
+
+    async previewAsync() {
+        if (this.isFormWithoutErrors()) {
+            const data = this.getFormValues();
+
+            // FIXME - navigate to campaign preview
+        } else {
+            this.showFormValidation();
+        }
+    }
+
+    render() {
+        const t = this.props.t;
+
+        const testUsersColumns = [
+            { data: 1, title: t('Email') },
+            { data: 4, title: t('List ID') },
+            { data: 5, title: t('List') },
+            { data: 6, title: t('Segment') },
+            { data: 7, title: t('List namespace') }
+        ];
+
+        return (
+            <Form stateOwner={this}>
+                <TableSelect id="testUser" label={t('Preview campaign as')} withHeader dropdown dataUrl={`rest/campaigns-test-users-table/${this.props.entity.id}`} columns={testUsersColumns} selectionLabelIndex={1} />
+                <ButtonRow>
+                    <Button className="btn-primary" label={t('Preview')} onClickAsync={::this.previewAsync}/>
+                </ButtonRow>
+            </Form>
+        );
+    }
+}
+
+@translate()
+@withForm
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+class SendControls extends Component {
+    constructor(props) {
+        super(props);
+        this.initForm();
+    }
+
+    static propTypes = {
+        entity: PropTypes.object.isRequired,
+        refreshEntity: PropTypes.func.isRequired
+    }
+
+    localValidateFormValues(state) {
+        const t = this.props.t;
+
+        state.setIn(['date', 'error'], null);
+        state.setIn(['time', 'error'], null);
+
+        if (!state.getIn(['sendLater', 'value'])) {
+            const dateValue = state.getIn(['date', 'value']);
+            if (!dateValue) {
+                state.setIn(['date', 'error'], t('Date must not be empty'));
+            } else if (!moment.utc(dateValue, 'YYYY-MM-DD').isValid()) {
+                state.setIn(['date', 'error'], t('Date is invalid'));
+            }
+
+            const timeValue = state.getIn(['time', 'value']);
+            const time = moment.utc(timeValue, 'HH:mm').isValid();
+            if (!timeValue) {
+                state.setIn(['time', 'error'], t('Time must not be empty'));
+            } else if (!time) {
+                state.setIn(['time', 'error'], t('Time is invalid'));
+            }
+        }
+    }
+
+    componentDidMount() {
+        const entity = this.props.entity;
+
+        if (entity.scheduled) {
+            const date = moment(entity.scheduled);
+            this.populateFormValues({
+                sendLater: true,
+                date: date.utc().format('YYYY-MM-DD'),
+                time: date.utc().format('HH:mm')
+            });
+
+        } else {
+            this.populateFormValues({
+                sendLater: false,
+                date: '',
+                time: ''
+            });
+        }
+    }
+
+    async refreshEntity() {
+        await this.props.refreshEntity();
+    }
+
+    async postAndMaskStateError(url) {
+        try {
+            await axios.post(getUrl(url));
+        } catch (err) {
+            if (err instanceof interoperableErrors.InvalidStateError) {
+                // Just mask the fact that it's not possible to start anything and refresh instead.
+            } else {
+                throw err;
+            }
+        }
+    }
+
+    async scheduleAsync() {
+        if (this.isFormWithoutErrors()) {
+            const data = this.getFormValues();
+            const date = moment.utc(data.date);
+            const time = moment.utc(date.time);
+
+            date.hour(time.hour());
+            date.minute(time.minute());
+            date.second(0);
+            date.millisecond(0);
+
+            await this.postAndMaskStateError(`rest/campaign-start-at/${this.props.entity.id}/${date.toDate()}`);
+
+        } else {
+            this.showFormValidation();
+        }
+
+        await this.refreshEntity();
+    }
+
+    async startAsync() {
+        await this.postAndMaskStateError(`rest/campaign-start/${this.props.entity.id}`);
+        await this.refreshEntity();
+    }
+
+    async stopAsync() {
+        await this.postAndMaskStateError(`rest/campaign-stop/${this.props.entity.id}`);
+        await this.refreshEntity();
+    }
+
+    async resetAsync() {
+        await this.postAndMaskStateError(`rest/campaign-reset/${this.props.entity.id}`);
+        await this.refreshEntity();
+    }
+
+    render() {
+        const t = this.props.t;
+        const entity = this.props.entity;
+
+        if (entity.status === CampaignStatus.IDLE || entity.status === CampaignStatus.PAUSED || (entity.status === CampaignStatus.SCHEDULED && entity.scheduled)) {
+
+            const subscrInfo = entity.subscriptionsTotal === undefined ? '' : ` (${entity.subscriptionsTotal} ${t('subscribers')})`;
+
+            return (
+                <div>
+                    <AlignedRow label={t('Send status')}>
+                        {entity.scheduled ? t('Campaign is scheduled for delivery.') : t('Campaign is ready to be sent out.')}
+                    </AlignedRow>
+
+                    <Form stateOwner={this}>
+                        <CheckBox id="sendLater" label={t('Send later')} text={t('Schedule deliver at particular date/time')}/>
+                        {this.getFormValue('sendLater') &&
+                            <div>
+                                <DatePicker id="date" label={t('Date')} />
+                                <InputField id="time" label={t('Time')} help={t('Enter 24-hour time in format HH:MM (e.g. 13:48)')}/>
+                            </div>
+                        }
+                    </Form>
+                    <ButtonRow className={campaignsStyles.sendButtonRow}>
+                        {this.getFormValue('sendLater') ?
+                            <Button className="btn-primary" icon="send" label={(entity.scheduled ? t('Reschedule send') : t('Schedule send')) + subscrInfo} onClickAsync={::this.scheduleAsync}/>
+                            :
+                            <Button className="btn-primary" icon="send" label={t('Send') + subscrInfo} onClickAsync={::this.startAsync}/>
+                        }
+                    </ButtonRow>
+                </div>
+            );
+
+        } else if (entity.status === CampaignStatus.SENDING || (entity.status === CampaignStatus.SCHEDULED && !entity.scheduled)) {
+            return (
+                <div>
+                    <div>
+                        {t('Campaign is being sent out.')}
+                    </div>
+                    <ButtonRow>
+                        <Button className="btn-primary" icon="stop" label={t('Stop')} onClickAsync={::this.pauseAsync}/>
+                    </ButtonRow>
+                </div>
+            );
+
+        } else if (entity.status === CampaignStatus.FINISHED) {
+            return (
+                <div>
+                    <div>
+                        {t('All messages sent! Hit "Continue" if you you want to send this campaign to new subscribers.')}
+                    </div>
+                    <ButtonRow>
+                        <Button className="btn-primary" icon="play" label={t('Continue')} onClickAsync={::this.startAsync}/>
+                        <Button className="btn-primary" icon="refresh" label={t('Reset')} onClickAsync={::this.resetAsync}/>
+                    </ButtonRow>
+                </div>
+            );
+        } else {
+
+            return null;
+        }
+    }
+}
+
+@translate()
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class Status extends Component {
+    constructor(props) {
+        super(props);
+
+        const t = props.t;
+
+        this.state = {
+            entity: props.entity,
+            sendConfiguration: null
+        };
+
+        const { campaignTypeLabels, campaignStatusLabels } = getCampaignLabels(t);
+        this.campaignTypeLabels = campaignTypeLabels;
+        this.campaignStatusLabels = campaignStatusLabels;
+
+        this.refreshTimeoutHandler = ::this.periodicRefreshTask;
+        this.refreshTimeoutId = 0;
+    }
+
+    static propTypes = {
+        entity: PropTypes.object
+    }
+
+    @withAsyncErrorHandler
+    async refreshEntity() {
+        let resp;
+
+        resp = await axios.get(getUrl(`rest/campaigns-stats/${this.props.entity.id}`));
+        const entity = resp.data;
+
+        resp = await axios.get(getUrl(`rest/send-configurations-public/${entity.send_configuration}`));
+        const sendConfiguration = resp.data;
+
+        this.setState({
+            entity,
+            sendConfiguration
+        });
+    }
+
+    async periodicRefreshTask() {
+        // The periodic task runs all the time, so that we don't have to worry about starting/stopping it as a reaction to the buttons.
+        await this.refreshEntity();
+        if (this.refreshTimeoutHandler) { // For some reason the task gets rescheduled if server is restarted while the page is shown. That why we have this check here.
+            this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
+        }
+    }
+
+    componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
+        this.periodicRefreshTask();
+    }
+
+    componentWillUnmount() {
+        clearTimeout(this.refreshTimeoutId);
+        this.refreshTimeoutHandler = null;
+    }
+
+
+    render() {
+        const t = this.props.t;
+        const entity = this.state.entity;
+
+        let sendSettings;
+        if (this.state.sendConfiguration) {
+            sendSettings = [];
+
+            const addOverridable = (id, label) => {
+                sendSettings.push(<AlignedRow key={id} label={label}>{entity[id + '_override'] === null ? this.state.sendConfiguration[id] : entity[id + '_override']}</AlignedRow>);
+            };
+
+            addOverridable('from_name', t('"From" name'));
+            addOverridable('from_email', t('"From" email address'));
+            addOverridable('reply_to', t('"Reply-to" email address'));
+            addOverridable('subject', t('"Subject" line'));
+        } else {
+            sendSettings =  <AlignedRow>{t('Loading send configuration ...')}</AlignedRow>
+        }
+
+        const listsColumns = [
+            { data: 1, title: t('Name') },
+            { data: 2, title: t('ID'), render: data => <code>{data}</code> },
+            { data: 3, title: t('Namespace') },
+            { data: 4, title: t('Segment') }
+        ];
+
+        return (
+            <div>
+                <Title>{t('Campaign Status')}</Title>
+
+                <AlignedRow label={t('Name')}>{entity.name}</AlignedRow>
+                <AlignedRow label={t('Subscribers')}>{entity.subscriptionsTotal === undefined ? t('computing ...') : entity.subscriptionsTotal}</AlignedRow>
+                <AlignedRow label={t('Status')}>{this.campaignStatusLabels[entity.status]}</AlignedRow>
+
+                {sendSettings}
+
+                <AlignedRow label={t('Target lists/segments')}>
+                    <Table withHeader dataUrl={`rest/lists-with-segment-by-campaign-table/${this.props.entity.id}`} columns={listsColumns} />
+                </AlignedRow>
+
+                <hr/>
+
+                <TestUser entity={entity}/>
+
+                <hr/>
+
+                <SendControls entity={entity} refreshEntity={::this.refreshEntity}/>
+            </div>
+        );
+    }
+}

client/src/campaigns/helpers.js

@@ -1,8 +1,11 @@
 'use strict';
 
-import {CampaignType} from "../../../shared/campaigns";
+import {
+    CampaignStatus,
+    CampaignType
+} from "../../../shared/campaigns";
 
-export function getCampaignTypeLabels(t) {
+export function getCampaignLabels(t) {
 
     const campaignTypeLabels = {
         [CampaignType.REGULAR]: t('Regular'),
@@ -10,5 +13,19 @@ export function getCampaignTypeLabels(t) {
         [CampaignType.RSS]: t('RSS')
     };
 
-    return campaignTypeLabels;
+    const campaignStatusLabels = {
+        [CampaignStatus.IDLE]: t('Idle'),
+        [CampaignStatus.SCHEDULED]: t('Scheduled'),
+        [CampaignStatus.PAUSED]: t('Paused'),
+        [CampaignStatus.FINISHED]: t('Finished'),
+        [CampaignStatus.PAUSED]: t('Paused'),
+        [CampaignStatus.INACTIVE]: t('Inactive'),
+        [CampaignStatus.ACTIVE]: t('Active')
+    };
+
+
+    return {
+        campaignStatusLabels,
+        campaignTypeLabels
+    };
 }

client/src/campaigns/root.js

@@ -2,6 +2,7 @@
 
 import React from 'react';
 
+import Status from './Status';
 import CampaignsCUD from './CUD';
 import Content from './Content';
 import CampaignsList from './List';
@@ -29,6 +30,12 @@ function getMenus(t) {
                     },
                     link: params => `/campaigns/${params.campaignId}/edit`,
                     navs: {
+                        status: {
+                            title: t('Status'),
+                            link: params => `/campaigns/${params.campaignId}/status`,
+                            visible: resolved => resolved.campaign.permissions.includes('viewStats'),
+                            panelRender: props => <Status entity={props.resolved.campaign} />
+                        },
                         ':action(edit|delete)': {
                             title: t('Edit'),
                             link: params => `/campaigns/${params.campaignId}/edit`,

client/src/campaigns/styles.scss

@@ -36,3 +36,7 @@
   text-align: right;
   margin-bottom: 15px;
 }
+
+.sendButtonRow {
+  margin-top: 10px;
+}

client/src/campaigns/triggers/CUD.js

@@ -30,7 +30,7 @@ import {
     Event
 } from '../../../../shared/triggers';
 import moment from 'moment';
-import {getCampaignTypeLabels} from "../helpers";
+import {getCampaignLabels} from "../helpers";
 
 
 @translate()
@@ -44,7 +44,7 @@ export default class CUD extends Component {
 
         this.state = {};
 
-        this.campaignTypes = getCampaignTypeLabels(props.t);
+        this.campaignTypeLabels = getCampaignLabels(props.t);
 
         const {entityLabels, eventLabels} = getTriggerTypes(props.t);
         this.entityLabels = entityLabels;
@@ -186,7 +186,7 @@ export default class CUD extends Component {
         const campaignsColumns = [
             { data: 1, title: t('Name') },
             { data: 2, title: t('Description') },
-            { data: 3, title: t('Type'), render: data => this.campaignTypes[data] },
+            { data: 3, title: t('Type'), render: data => this.campaignTypeLabels[data] },
             { data: 4, title: t('Created'), render: data => moment(data).fromNow() },
             { data: 5, title: t('Namespace') }
         ];

client/src/lib/bootstrap-components.js

@@ -249,6 +249,7 @@ class ModalDialog extends Component {
         // are capture, converted to onClose callback and prevented. It's up to the parent to decide whether to
         // hide the modal or not.
         if (!this.props.hidden) {
+            // noinspection JSIgnoredPromiseFromCall
             this.onClose();
             evt.preventDefault();
         }

client/src/lib/page.js

@@ -185,6 +185,7 @@ class RouteContent extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.resolve(this.props);
     }
 
@@ -193,6 +194,7 @@ class RouteContent extends Component {
 
     componentWillReceiveProps(nextProps) {
         if (this.props.match.params !== nextProps.match.params && needsResolve(this.props.route, nextProps.route, this.props.match, nextProps.match)) {
+            // noinspection JSIgnoredPromiseFromCall
             this.resolve(nextProps);
         }
     }
@@ -279,6 +281,7 @@ class SectionContent extends Component {
         }
 
         this.historyUnlisten = props.history.listen((location, action) => {
+            // noinspection JSIgnoredPromiseFromCall
             this.closeFlashMessage();
         })
     }

client/src/lib/table.js

@@ -154,6 +154,7 @@ class Table extends Component {
                 }
             }
 
+            // noinspection JSIgnoredPromiseFromCall
             this.notifySelection(this.props.onSelectionDataAsync, this.selectionMap);
         }
     }
@@ -278,6 +279,7 @@ class Table extends Component {
 
                 if (self.props.selectMode === TableSelectMode.SINGLE) {
                     if (selectionMap.size !== 1 || !selectionMap.has(rowKey)) {
+                        // noinspection JSIgnoredPromiseFromCall
                         self.notifySelection(self.props.onSelectionChangedAsync, new Map([[rowKey, data]]));
                     }
 
@@ -290,6 +292,7 @@ class Table extends Component {
                         newSelMap.set(rowKey, data);
                     }
 
+                    // noinspection JSIgnoredPromiseFromCall
                     self.notifySelection(self.props.onSelectionChangedAsync, newSelMap);
                 }
             });
@@ -321,6 +324,7 @@ class Table extends Component {
            clearTimeout(this.refreshTimeoutId);
         });
 
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchAndNotifySelectionData();
     }
 
@@ -344,6 +348,8 @@ class Table extends Component {
         });
 
         this.updateSelectInfo();
+
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchAndNotifySelectionData();
     }
 
@@ -375,6 +381,8 @@ class Table extends Component {
 
     async deselectAll(evt) {
         evt.preventDefault();
+
+        // noinspection JSIgnoredPromiseFromCall
         this.notifySelection(this.props.onSelectionChangedAsync, new Map());
     }
 

client/src/lib/tree.js

@@ -82,6 +82,7 @@ class TreeTable extends Component {
                 treeData: nextProps.data
             });
         } else if (nextProps.dataUrl && this.props.dataUrl !== nextProps.dataUrl) {
+            // noinspection JSIgnoredPromiseFromCall
             this.loadData(next.props.dataUrl);
         }
     }
@@ -107,6 +108,7 @@ class TreeTable extends Component {
 
     componentDidMount() {
         if (!this.props.data && this.props.dataUrl) {
+            // noinspection JSIgnoredPromiseFromCall
             this.loadData(this.props.dataUrl);
         }
 
@@ -229,6 +231,7 @@ class TreeTable extends Component {
         const selection = this.destringifyKey(this.tree.getActiveNode().key);
 
         if (selection !== this.props.selection) {
+            // noinspection JSIgnoredPromiseFromCall
             this.onSelectionChanged(selection);
         }
     }
@@ -252,6 +255,7 @@ class TreeTable extends Component {
         }
 
         if (updated) {
+            // noinspection JSIgnoredPromiseFromCall
             this.onSelectionChanged(selection);
         }
     }

client/src/lib/untrusted.js

@@ -125,6 +125,7 @@ export class UntrustedContentHost extends Component {
 
     scheduleRefreshAccessToken() {
         this.refreshAccessTokenTimeout = setTimeout(() => {
+            // noinspection JSIgnoredPromiseFromCall
             this.refreshAccessToken();
             this.scheduleRefreshAccessToken();
         }, 60 * 1000);
@@ -136,6 +137,7 @@ export class UntrustedContentHost extends Component {
         }
 
         if (!this.state.hasAccessToken) {
+            // noinspection JSIgnoredPromiseFromCall
             this.refreshAccessToken();
         }
     }

client/src/lists/List.js

@@ -36,6 +36,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/lists/forms/List.js

@@ -35,6 +35,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/lists/imports/CUD.js

@@ -27,7 +27,7 @@ import {
     withErrorHandling
 } from '../../lib/error-handling';
 import {DeleteModalDialog} from "../../lib/modals";
-import {getImportTypes} from './helpers';
+import {getImportLabels} from './helpers';
 import {
     ImportSource,
     inProgress,
@@ -62,7 +62,7 @@ export default class CUD extends Component {
 
         this.state = {};
 
-        const {importSourceLabels, mappingTypeLabels} = getImportTypes(props.t);
+        const {importSourceLabels, mappingTypeLabels} = getImportLabels(props.t);
 
         this.importSourceLabels = importSourceLabels;
 

client/src/lists/imports/List.js

@@ -12,7 +12,7 @@ import {
 } from '../../lib/page';
 import {withErrorHandling} from '../../lib/error-handling';
 import {Table} from '../../lib/table';
-import {getImportTypes} from './helpers';
+import {getImportLabels} from './helpers';
 import {Icon} from "../../lib/bootstrap-components";
 import mailtrainConfig from 'mailtrainConfig';
 import moment from "moment";
@@ -28,7 +28,7 @@ export default class List extends Component {
 
         this.state = {};
 
-        const {importSourceLabels, importStatusLabels} = getImportTypes(props.t);
+        const {importSourceLabels, importStatusLabels} = getImportLabels(props.t);
         this.importSourceLabels = importSourceLabels;
         this.importStatusLabels = importStatusLabels;
     }

client/src/lists/imports/RunStatus.js

@@ -13,7 +13,7 @@ import {
     withAsyncErrorHandler,
     withErrorHandling
 } from '../../lib/error-handling';
-import {getImportTypes} from './helpers';
+import {getImportLabels} from './helpers';
 import axios from "../../lib/axios";
 import {getUrl} from "../../lib/urls";
 import moment from "moment";
@@ -32,7 +32,7 @@ export default class Status extends Component {
             entity: props.entity
         };
 
-        const {importSourceLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
+        const {importSourceLabels, importStatusLabels, runStatusLabels} = getImportLabels(props.t);
         this.importSourceLabels = importSourceLabels;
         this.importStatusLabels = importStatusLabels;
         this.runStatusLabels = runStatusLabels;
@@ -69,6 +69,7 @@ export default class Status extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.periodicRefreshTask();
     }
 

client/src/lists/imports/Status.js

@@ -17,7 +17,7 @@ import {
     withAsyncErrorHandler,
     withErrorHandling
 } from '../../lib/error-handling';
-import {getImportTypes} from './helpers';
+import {getImportLabels} from './helpers';
 import {
     prepFinishedAndNotInProgress,
     runInProgress,
@@ -46,7 +46,7 @@ export default class Status extends Component {
             entity: props.entity
         };
 
-        const {importSourceLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
+        const {importSourceLabels, importStatusLabels, runStatusLabels} = getImportLabels(props.t);
         this.importSourceLabels = importSourceLabels;
         this.importStatusLabels = importStatusLabels;
         this.runStatusLabels = runStatusLabels;
@@ -77,6 +77,7 @@ export default class Status extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.periodicRefreshTask();
     }
 

client/src/lists/imports/helpers.js

@@ -3,7 +3,7 @@
 import React from 'react';
 import {ImportSource, MappingType, ImportStatus, RunStatus} from '../../../../shared/imports';
 
-export function getImportTypes(t) {
+export function getImportLabels(t) {
 
     const importSourceLabels = {
         [ImportSource.CSV_FILE]: t('CSV file'),

client/src/namespaces/CUD.js

@@ -85,6 +85,7 @@ export default class CUD extends Component {
         }
 
         if (!this.isEditGlobal()) {
+            // noinspection JSIgnoredPromiseFromCall
             this.loadTreeData();
         }
     }

client/src/namespaces/List.js

@@ -35,6 +35,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/reports/CUD.js

@@ -50,6 +50,7 @@ export default class CUD extends Component {
             state.formState = state.formState.setIn(['data', 'user_fields', 'value'], '');
 
             if (newVal) {
+                // noinspection JSIgnoredPromiseFromCall
                 this.fetchUserFields(newVal);
             }
         }

client/src/reports/List.js

@@ -51,6 +51,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/reports/Output.js

@@ -34,6 +34,7 @@ export default class Output extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.loadOutput();
     }
 

client/src/reports/View.js

@@ -35,6 +35,7 @@ export default class View extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.loadContent();
     }
 

client/src/reports/templates/List.js

@@ -37,6 +37,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/send-configurations/List.js

@@ -49,6 +49,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/templates/List.js

@@ -56,6 +56,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

client/src/templates/mosaico/List.js

@@ -40,6 +40,7 @@ export default class List extends Component {
     }
 
     componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
         this.fetchPermissions();
     }
 

config/default.yaml

@@ -129,7 +129,7 @@ nodemailer:
 
 queue:
   # How many parallel sender processes to spawn
-  processes: 1
+  processes: 2
 
 cors:
   # Allow subscription widgets to be embedded
@@ -229,7 +229,7 @@ roles:
     master:
       name: Master
       description: All permissions
-      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats]
+      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, manageMessages]
 
   template:
     master:

index.js

@@ -7,7 +7,7 @@ const http = require('http');
 //const triggers = require('./services/triggers');
 const importer = require('./lib/importer');
 const feedcheck = require('./lib/feedcheck');
-// const verpServer = require('./services/verp-server');
+const verpServer = require('./services/verp-server');
 const testServer = require('./services/test-server');
 //const postfixBounceServer = require('./services/postfix-bounce-server');
 const tzupdate = require('./services/tzupdate');
@@ -83,7 +83,7 @@ dbcheck(err => { // Check if database needs upgrading before starting the server
     .then(() =>
         executor.spawn(() => {
             testServer(() => {
-                //verpServer(() => {
+                verpServer(() => {
                     startHTTPServer(true, trustedPort, () => {
                         startHTTPServer(false, sandboxPort, () => {
                             privilegeHelpers.dropRootPrivileges();
@@ -106,7 +106,7 @@ dbcheck(err => { // Check if database needs upgrading before starting the server
                             });
                         });
                     });
-                //});
+                });
             });
         })
     );

lib/knex.js

@@ -8,7 +8,7 @@ const knex = require('knex')({
     migrations: {
         directory: __dirname + '/../setup/knex/migrations'
     }
-    //, debug: true
+    , debug: true
 });
 
 module.exports = knex;

lib/mailers.js

@@ -230,5 +230,13 @@ async function _createTransport(sendConfiguration) {
     return transport;
 }
 
+class MailerError extends Error {
+    constructor(msg, responseCode) {
+        super(msg);
+        this.responseCode = responseCode;
+    }
+}
+
 module.exports.getOrCreateMailer = getOrCreateMailer;
 module.exports.invalidateMailer = invalidateMailer;
+module.exports.MailerError = MailerError;

lib/senders.js

@@ -3,6 +3,8 @@
 const fork = require('child_process').fork;
 const log = require('npmlog');
 const path = require('path');
+const knex = require('../lib/knex');
+const {CampaignStatus} = require('../shared/campaigns');
 
 let messageTid = 0;
 let senderProcess;
@@ -10,6 +12,10 @@ let senderProcess;
 function spawn(callback) {
     log.verbose('Senders', 'Spawning master sender process');
 
+    knex.transaction(async tx => {
+        await tx('campaigns').where('status', CampaignStatus.SENDING).update({status: CampaignStatus.SCHEDULED});
+
+    }).then(() => {
         senderProcess = fork(path.join(__dirname, '..', 'services', 'sender-master.js'), [], {
             cwd: path.join(__dirname, '..'),
             env: {NODE_ENV: process.env.NODE_ENV}
@@ -27,11 +33,12 @@ function spawn(callback) {
         senderProcess.on('close', (code, signal) => {
             log.error('Senders', 'Master sender process exited with code %s signal %s', code, signal);
         });
+    });
 }
 
 function scheduleCheck() {
     senderProcess.send({
-        type: 'scheduleCheck',
+        type: 'schedule-check',
         tid: messageTid
     });
 
@@ -40,7 +47,7 @@ function scheduleCheck() {
 
 function reloadConfig(sendConfigurationId) {
     senderProcess.send({
-        type: 'reloadConfig',
+        type: 'reload-config',
         data: {
             sendConfigurationId
         },

models/blacklist.js

@@ -69,11 +69,9 @@ async function serverValidate(context, data) {
     return result;
 }
 
-module.exports = {
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.add = add;
-    add,
+module.exports.remove = remove;
-    remove,
+module.exports.search = search;
-    search,
+module.exports.isBlacklisted = isBlacklisted;
-    isBlacklisted,
+module.exports.serverValidate = serverValidate;
-    serverValidate
-};

models/campaigns.js

@@ -10,10 +10,13 @@ const shares = require('./shares');
 const namespaceHelpers = require('../lib/namespace-helpers');
 const files = require('./files');
 const templates = require('./templates');
-const { CampaignSource, CampaignType, getSendConfigurationPermissionRequiredForSend} = require('../shared/campaigns');
+const { CampaignStatus, CampaignSource, CampaignType, getSendConfigurationPermissionRequiredForSend} = require('../shared/campaigns');
-const segments = require('./segments');
 const sendConfigurations = require('./send-configurations');
 const triggers = require('./triggers');
+const {SubscriptionStatus} = require('../shared/lists');
+const subscriptions = require('./subscriptions');
+const segments = require('./segments');
+const senders = require('../lib/senders');
 
 const allowedKeysCommon = ['name', 'description', 'segment', 'namespace',
     'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url'];
@@ -25,9 +28,11 @@ const Content = {
     ALL: 0,
     WITHOUT_SOURCE_CUSTOM: 1,
     ONLY_SOURCE_CUSTOM: 2,
-    RSS_ENTRY: 3
+    RSS_ENTRY: 3,
+    SETTINGS_WITH_STATS: 4
 };
 
+
 function hash(entity, content) {
     let filteredEntity;
 
@@ -88,6 +93,67 @@ async function listOthersWhoseListsAreIncludedDTAjax(context, campaignId, listId
     );
 }
 
+async function listTestUsersDTAjax(context, campaignId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'view');
+
+        const subscriptionsQueries = [];
+        const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+
+        for (const cpgList of cpgLists) {
+            const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {
+            };
+            const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
+
+            subscriptionsQueries.push(function () {
+                this.from(subsTable)
+                    .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+                    .where(subsTable + '.is_test', true)
+                    .where(function() {
+                        addSegmentQuery(this);
+                    })
+                    .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('? AS list', [cpgList.list]), knex.raw('? AS segment', [cpgList.segment])]);
+            });
+        }
+
+        if (subscriptionsQueries.length > 0) {
+            return await dtHelpers.ajaxListWithPermissions(
+                context,
+                [{ entityTypeId: 'list', requiredOperations: ['viewSubscriptions'] }],
+                params,
+                builder => {
+                    let ret;
+                    if (subscriptionsQueries.length > 1) {
+                        ret = builder.unionAll(subscriptionsQueries, true)
+                            .as('test_subscriptions');
+                    } else {
+                        ret = builder.from(function () { subscriptionsQueries[0].apply(this); this.as('test_subscriptions'); })
+                            .as('test_subscriptions');
+                    }
+
+                    ret = ret
+                        .innerJoin('lists', 'test_subscriptions.list', 'lists.id')
+                        .innerJoin('segments', 'test_subscriptions.segment', 'segments.id')
+                        .innerJoin('namespaces', 'lists.namespace', 'namespaces.id');
+
+                    return ret;
+                },
+                ['test_subscriptions.campaign_list_id', 'test_subscriptions.email', 'test_subscriptions.list', 'test_subscriptions.segment', 'lists.cid', 'lists.name', 'segments.name', 'namespaces.name']
+            );
+
+        } else {
+            const result = {
+                draw: params.draw,
+                recordsTotal: 0,
+                recordsFiltered: 0,
+                data: []
+            };
+
+            return result;
+        }
+    });
+}
+
 async function rawGetByIdTx(tx, id) {
     const entity = await tx('campaigns').where('campaigns.id', id)
         .innerJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
@@ -121,7 +187,27 @@ async function getByIdTx(tx, context, id, withPermissions = true, content = Cont
 
     let entity = await rawGetByIdTx(tx, id);
 
-    if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+    if (content === Content.ALL || content === Content.RSS_ENTRY) {
+        // Return everything
+
+    } else if (content === Content.SETTINGS_WITH_STATS) {
+        delete entity.data.sourceCustom;
+
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
+
+        const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id, true);
+        if (unsentQryGen) {
+            const res = await unsentQryGen(tx).count('* AS subscriptionsToSend').first();
+            entity.subscriptionsToSend = res.subscriptionsToSend;
+        }
+
+        const totalQryGen = await getSubscribersQueryGeneratorTx(tx, id, false);
+        if (totalQryGen) {
+            const res = await totalQryGen(tx).count('* AS subscriptionsTotal').first();
+            entity.subscriptionsTotal = res.subscriptionsTotal;
+        }
+
+    } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
         delete entity.data.sourceCustom;
 
     } else if (content === Content.ONLY_SOURCE_CUSTOM) {
@@ -251,37 +337,6 @@ async function _createTx(tx, context, entity, content) {
 
         await tx('campaign_lists').insert(entity.lists.map(x => ({campaign: id, ...x})));
 
-        await knex.schema.raw('CREATE TABLE `campaign__' + id + '` (\n' +
-            '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
-            '  `list` int(10) unsigned NOT NULL,\n' +
-            '  `segment` int(10) unsigned NOT NULL,\n' +
-            '  `subscription` int(10) unsigned NOT NULL,\n' +
-            '  `status` tinyint(4) unsigned NOT NULL DEFAULT \'0\',\n' +
-            '  `response` varchar(255) DEFAULT NULL,\n' +
-            '  `response_id` varchar(255) CHARACTER SET ascii DEFAULT NULL,\n' +
-            '  `updated` timestamp NULL DEFAULT NULL,\n' +
-            '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
-            '  PRIMARY KEY (`id`),\n' +
-            '  UNIQUE KEY `list` (`list`,`segment`,`subscription`),\n' +
-            '  KEY `created` (`created`),\n' +
-            '  KEY `response_id` (`response_id`),\n' +
-            '  KEY `status_index` (`status`),\n' +
-            '  KEY `subscription_index` (`subscription`)\n' +
-            ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
-
-        await knex.schema.raw('CREATE TABLE `campaign__tracker' + id + '` (\n' +
-            '  `list` int(10) unsigned NOT NULL,\n' +
-            '  `subscriber` int(10) unsigned NOT NULL,\n' +
-            '  `link` int(10) NOT NULL,\n' +
-            '  `ip` varchar(100) CHARACTER SET ascii DEFAULT NULL,\n' +
-            '  `device_type` varchar(50) DEFAULT NULL,\n' +
-            '  `country` varchar(2) CHARACTER SET ascii DEFAULT NULL,\n' +
-            '  `count` int(11) unsigned NOT NULL DEFAULT \'1\',\n' +
-            '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
-            '  PRIMARY KEY (`list`,`subscriber`,`link`),\n' +
-            '  KEY `created_index` (`created`)\n' +
-            ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
-
         await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: id });
 
         if (copyFilesFrom) {
@@ -351,13 +406,16 @@ async function remove(context, id) {
     await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'delete');
 
+        const existing = tx('campaigns').where('id', id);
+        if (existing.status === CampaignStatus.SENDING) {
+            return new interoperableErrors.InvalidStateError;
+        }
+
         // FIXME - deal with deletion of dependent entities (files)
 
         await triggers.removeAllByCampaignIdTx(tx, context, id);
 
         await tx('campaigns').where('id', id).del();
-        await knex.schema.dropTableIfExists('campaign__' + id);
-        await knex.schema.dropTableIfExists('campaign_tracker__' + id);
     });
 }
 
@@ -371,18 +429,253 @@ async function enforceSendPermissionTx(tx, context, campaignId) {
     await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
 }
 
-// This is to handle circular dependency with triggers.js
+
-Object.assign(module.exports, {
+// Message API
-    Content,
+
-    hash,
+function getMessageCid(campaignCid, listCid, subscriptionCid) {
-    listDTAjax,
+    return [campaignCid, listCid, subscriptionCid].join('.')
-    listWithContentDTAjax,
+}
-    listOthersWhoseListsAreIncludedDTAjax,
+
-    getByIdTx,
+async function getMessageByCid(messageCid) {
-    getById,
+    const messageCid = messageCid.split('.');
-    create,
+
-    createRssTx,
+    if (messageCid.length !== 3) {
-    updateWithConsistencyCheck,
+        return null;
-    remove,
+    }
-    enforceSendPermissionTx
+
+    const [campaignCid, listCid, subscriptionCid] = messageCid;
+
+    await knex.transaction(async tx => {
+        const list = await tx('lists').where('cid', listCid).select('id');
+        const subscrTblName = subscriptions.getSubscriptionTableName(list.id);
+
+        const message = await tx('campaign_messages')
+            .innerJoin('campaigns', 'campaign_messages.campaign', 'campaigns.id')
+            .innerJoin(subscrTblName, subscrTblName + '.id', 'campaign_messages.subscription')
+            .leftJoin('segments', 'segment.id', 'campaign_messages.segment') // This is just to make sure that the respective segment still exists or return null if it doesn't
+            .leftJoin('send_configurations', 'send_configurations.id', 'campaign_messages.send_configuration') // This is just to make sure that the respective send_configuration still exists or return null if it doesn't
+            .where(subscrTblName + '.cid', subscriptionCid)
+            .where('campaigns.cid', campaignCid)
+            .select([
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'segments.id AS segment', 'campaign_messages.subscription',
+                'send_configurations.id AS send_configuration', 'campaign_messages.status', 'campaign_messages.response', 'campaign_messages.response_id',
+                'campaign_messages.updated', 'campaign_messages.created', 'send_configurations.verp_hostname AS verp_hostname'
+            ]);
+
+        if (message) {
+            await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+        }
+
+        return message;
     });
+}
+
+async function getMessageByResponseId(responseId) {
+    await knex.transaction(async tx => {
+        const message = await tx('campaign_messages')
+            .leftJoin('segments', 'segment.id', 'campaign_messages.segment') // This is just to make sure that the respective segment still exists or return null if it doesn't
+            .leftJoin('send_configurations', 'send_configurations.id', 'campaign_messages.send_configuration') // This is just to make sure that the respective send_configuration still exists or return null if it doesn't
+            .where('campaign_messages.response_id', responseId)
+            .select([
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'segments.id AS segment', 'campaign_messages.subscription',
+                'send_configurations.id AS send_configuration', 'campaign_messages.status', 'campaign_messages.response', 'campaign_messages.response_id',
+                'campaign_messages.updated', 'campaign_messages.created', 'send_configurations.verp_hostname AS verp_hostname'
+            ]);
+
+        if (message) {
+            await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+        }
+
+        return message;
+    });
+}
+
+const statusFieldMapping = {
+    [SubscriptionStatus.UNSUBSCRIBED]: 'unsubscribed',
+    [SubscriptionStatus.BOUNCED]: 'bounced',
+    [SubscriptionStatus.COMPLAINED]: 'complained'
+};
+
+async function _changeStatusByMessageTx(tx, context, message, subscriptionStatus) {
+    enforce(subscriptionStatus !== SubscriptionStatus.SUBSCRIBED);
+
+    if (message.status === SubscriptionStatus.SUBSCRIBED) {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+
+        if (!subscriptionStatus in statusFieldMapping) {
+            throw new Error('Unrecognized message status');
+        }
+
+        const statusField = statusFieldMapping[subscriptionStatus];
+
+        if (message.status === SubscriptionStatus.SUBSCRIBED) {
+            await tx('campaigns').increment(statusField, 1).where('id', message.campaign);
+        }
+
+        await tx('campaign_messages')
+            .where('id', message.id)
+            .update({
+                status: subscriptionStatus,
+                updated: knex.fn.now()
+            });
+    }
+}
+
+async function changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaignCid, listId, subscriptionId, subscriptionStatus) {
+    const campaign = await tx('campaigns').where('cid', campaignCid);
+    const message = await tx('campaign_messages')
+        .innerJoin('campaigns', 'campaign_messages.campaign', 'campaigns.id')
+        .where('campaigns.cid', campaignCid)
+        .where({subscription: subscriptionId, list: listId});
+
+    if (!message) {
+        throw new Error('Invalid campaign.')
+    }
+
+    await _changeStatusByMessageTx(tx, context, message, subscriptionStatus);
+}
+
+async function changeStatusByMessage(context, message, subscriptionStatus, updateSubscription) {
+    await knex.transaction(async tx => {
+        if (updateSubscription) {
+            await subscriptions.changeStatusTx(tx, context, message.list, message.subscription, subscriptionStatus);
+        }
+
+        await _changeStatusByMessageTx(tx, context, message, subscriptionStatus);
+    });
+}
+
+async function getSubscribersQueryGeneratorTx(tx, campaignId, onlyUnsent, batchSize) {
+    const subscriptionsQueries = [];
+    const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+
+    for (const cpgList of cpgLists) {
+        const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
+        const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
+
+        subscriptionsQueries.push(function() {
+            this.from(subsTable)
+                .leftJoin('campaign_messages', 'campaign_messages.subscription', subsTable + '.id')
+                .where('campaign_messages.campaign', cpgList.campaign)
+                .where('campaign_messages.list', cpgList.list)
+                .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+                .where(function() {
+                    addSegmentQuery(this);
+                })
+                .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('campaign_messages.id IS NOT NULL AS sent')]);
+        });
+    }
+
+    if (subscriptionsQueries.length > 0) {
+        return knx => knx.from('campaign_lists')
+            .where('campaign_lists.campaign', campaignId)
+            .innerJoin(
+                function () {
+                    let ret;
+                    if (subscriptionsQueries.length > 1) {
+                        ret = this.unionAll(subscriptionsQueries, true)
+                            .groupBy('email')
+                            .select(['email']).min('campaign_list_id AS campaign_list_id')
+                            .select(['sent']).max('sent AS sent');
+                    } else {
+                        ret = this.from(function () { subscriptionsQueries[0].apply(this); this.as('pending_subscriptions'); })
+                            .select(['email', 'sent', 'campaign_list_id']);
+                    }
+
+                    ret = ret.where('sent', false)
+                        .as('pending_subscriptions');
+
+                    if (batchSize) {
+                        ret = ret.limit(retrieveBatchSize);
+                    }
+
+                    return ret;
+                },
+                'campaign_lists.id',
+                'pending_subscriptions.campaign_list_id'
+            );
+
+    } else {
+        return null;
+    }
+}
+
+async function _changeStatus(context, campaignId, permittedCurrentStates, newState, invalidStateMessage, scheduled = null) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+
+        const entity = await tx('campaigns').where('id', campaignId).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!permittedCurrentStates.includes(entity.status)) {
+            throw new interoperableErrors.InvalidStateError(invalidStateMessage);
+        }
+
+        await tx('campaigns').where('id', campaignId).update({
+            status: newState,
+            scheduled
+        });
+    });
+
+    senders.scheduleCheck();
+}
+
+
+async function start(context, campaignId, startAt) {
+    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.PAUSED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt);
+}
+
+async function stop(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED], CampaignStatus.PAUSED, 'Cannot stop campaign until it is in SCHEDULED state');
+}
+
+async function reset(context, campaignId) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+
+        const entity = await tx('campaigns').where('id', campaignId).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (entity.status !== CampaignStatus.FINISHED && entity.status !== CampaignStatus.PAUSED) {
+            throw new interoperableErrors.InvalidStateError('Cannot reset campaign until it is FINISHED or PAUSED state');
+        }
+
+        await tx('campaigns').where('id', campaignId).update({
+            status: CampaignStatus.IDLE
+        });
+
+        await tx('campaign_messages').where('campaign', campaignId).del();
+        await tx('campaign_links').where('campaign', campaignId).del();
+    });
+}
+
+module.exports.Content = Content;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listWithContentDTAjax = listWithContentDTAjax;
+module.exports.listOthersWhoseListsAreIncludedDTAjax = listOthersWhoseListsAreIncludedDTAjax;
+module.exports.listTestUsersDTAjax = listTestUsersDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.createRssTx = createRssTx;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.enforceSendPermissionTx = enforceSendPermissionTx;
+
+module.exports.getMessageCid = getMessageCid;
+module.exports.getMessageByCid = getMessageByCid;
+module.exports.getMessageByResponseId = getMessageByResponseId;
+
+module.exports.changeStatusByCampaignCidAndSubscriptionIdTx = changeStatusByCampaignCidAndSubscriptionIdTx;
+module.exports.changeStatusByMessage = changeStatusByMessage;
+
+module.exports.getSubscribersQueryGeneratorTx = getSubscribersQueryGeneratorTx;
+
+module.exports.start = start;
+module.exports.stop = stop;
+module.exports.reset = reset;

models/confirmations.js

@@ -45,7 +45,5 @@ async function takeConfirmation(cid) {
     });
 }
 
-module.exports = {
+module.exports.addConfirmation = addConfirmation;
-    addConfirmation,
+module.exports.takeConfirmation = takeConfirmation;
-    takeConfirmation
-};

models/fields.js

@@ -724,26 +724,23 @@ function fromImport(listId, flds, data) { // assumes ungrouped subscription and
     return _fromText(listId, data, flds, true, 'column', false);
 }
 
-// This is to handle circular dependency with segments.js
+module.exports.Cardinality = Cardinality;
-Object.assign(module.exports, {
+module.exports.getFieldType = getFieldType;
-    Cardinality,
+module.exports.hash = hash;
-    getFieldType,
+module.exports.getById = getById;
-    hash,
+module.exports.list = list;
-    getById,
+module.exports.listTx = listTx;
-    list,
+module.exports.listGrouped = listGrouped;
-    listTx,
+module.exports.listGroupedTx = listGroupedTx;
-    listGrouped,
+module.exports.listByOrderListTx = listByOrderListTx;
-    listGroupedTx,
+module.exports.listDTAjax = listDTAjax;
-    listByOrderListTx,
+module.exports.listGroupedDTAjax = listGroupedDTAjax;
-    listDTAjax,
+module.exports.create = create;
-    listGroupedDTAjax,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    create,
+module.exports.remove = remove;
-    updateWithConsistencyCheck,
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
-    remove,
+module.exports.serverValidate = serverValidate;
-    removeAllByListIdTx,
+module.exports.forHbs = forHbs;
-    serverValidate,
+module.exports.fromPost = fromPost;
-    forHbs,
+module.exports.fromAPI = fromAPI;
-    fromPost,
+module.exports.fromImport = fromImport;
-    fromAPI,
-    fromImport
-});

models/files.js

@@ -302,18 +302,16 @@ async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toTyp
 }
 
 
-module.exports = {
+module.exports.filesDir = filesDir;
-    filesDir,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.list = list;
-    list,
+module.exports.getFileById = getFileById;
-    getFileById,
+module.exports.getFileByFilename = getFileByFilename;
-    getFileByFilename,
+module.exports.getFileByUrl = getFileByUrl;
-    getFileByUrl,
+module.exports.getFileByOriginalName = getFileByOriginalName;
-    getFileByOriginalName,
+module.exports.createFiles = createFiles;
-    createFiles,
+module.exports.removeFile = removeFile;
-    removeFile,
+module.exports.getFileUrl = getFileUrl;
-    getFileUrl,
+module.exports.getFilePath = getFilePath;
-    getFilePath,
+module.exports.copyAllTx = copyAllTx;
-    copyAllTx,
+module.exports.ReplacementBehavior = ReplacementBehavior;
-    ReplacementBehavior
-};

models/forms.js

@@ -265,13 +265,11 @@ function checkForMjmlErrors(form) {
     return errors;
 }
 
-module.exports = {
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.hash = hash;
-    hash,
+module.exports.getById = getById;
-    getById,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove,
+module.exports.getDefaultCustomFormValues = getDefaultCustomFormValues;
-    getDefaultCustomFormValues,
+module.exports.serverValidate = serverValidate;
-    serverValidate
-};

models/import-runs.js

@@ -62,8 +62,6 @@ async function listFailedDTAjax(context, listId, importId, importRunId, params)
 
 
 
-module.exports = {
+module.exports.getById = getById;
-    getById,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.listFailedDTAjax = listFailedDTAjax;
-    listFailedDTAjax
-};

models/imports.js

@@ -238,16 +238,13 @@ async function stop(context, listId, id) {
 }
 
 
-// This is to handle circular dependency with segments.js
+module.exports.filesDir = filesDir;
-module.exports = {
+module.exports.hash = hash;
-    filesDir,
+module.exports.getById = getById;
-    hash,
+module.exports.listDTAjax = listDTAjax;
-    getById,
+module.exports.create = create;
-    listDTAjax,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    create,
+module.exports.remove = remove;
-    updateWithConsistencyCheck,
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
-    remove,
+module.exports.start = start;
-    removeAllByListIdTx,
+module.exports.stop = stop;
-    start,
-    stop
-};

models/lists.js

@@ -45,6 +45,22 @@ async function listDTAjax(context, params) {
     );
 }
 
+async function listWithSegmentByCampaignDTAjax(context, campaignId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('campaign_lists', 'campaign_lists.list', 'lists.id')
+            .leftJoin('segments', 'segments.id', 'campaign_lists.segment')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace')
+            .where('campaign_lists.campaign', campaignId)
+            .orderBy('campaign_lists.id', 'asc'),
+        ['lists.id', 'lists.name', 'lists.cid', 'namespaces.name', 'segments.name']
+    );
+}
+
 async function _getByIdTx(tx, context, id) {
     await shares.enforceEntityPermissionTx(tx, context, 'list', id, 'view');
     const entity = await tx('lists').where('id', id).first();
@@ -192,16 +208,15 @@ async function getMergeTags(context, id) {
 }
 
 
-module.exports = {
+module.exports.UnsubscriptionMode = UnsubscriptionMode;
-    UnsubscriptionMode,
+module.exports.hash = hash;
-    hash,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.listWithSegmentByCampaignDTAjax = listWithSegmentByCampaignDTAjax;
-    getById,
+module.exports.getById = getById;
-    getByIdWithListFields,
+module.exports.getByIdWithListFields = getByIdWithListFields;
-    getByCid,
+module.exports.getByCid = getByCid;
-    create,
+module.exports.create = create;
-    updateWithConsistencyCheck,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    remove,
+module.exports.remove = remove;
-    removeFormFromAllTx,
+module.exports.removeFormFromAllTx = removeFormFromAllTx;
-    getMergeTags
+module.exports.getMergeTags = getMergeTags;
-};

models/mosaico-templates.js

@@ -97,11 +97,9 @@ async function remove(context, id) {
     });
 }
 
-module.exports = {
+module.exports.hash = hash;
-    hash,
+module.exports.getById = getById;
-    getById,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove
-};

models/namespaces.js

@@ -187,11 +187,9 @@ async function remove(context, id) {
     });
 }
 
-module.exports = {
+module.exports.hash = hash;
-    hash,
+module.exports.listTree = listTree;
-    listTree,
+module.exports.getById = getById;
-    getById,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove
-};

models/report-templates.js

@@ -91,12 +91,10 @@ async function getUserFieldsById(context, id) {
     });
 }
 
-module.exports = {
+module.exports.hash = hash;
-    hash,
+module.exports.getById = getById;
-    getById,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove,
+module.exports.getUserFieldsById = getUserFieldsById;
-    getUserFieldsById
-};

models/reports.js

@@ -147,13 +147,13 @@ async function bulkChangeState(oldState, newState) {
 
 
 const campaignFieldsMapping = {
-    tracker_count: 'tracker.count',
+    tracker_count: 'campaign_links.count',
-    country: 'tracker.country',
+    country: 'campaign_links.country',
-    device_type: 'tracker.device_type',
+    device_type: 'campaign_links.device_type',
-    status: 'campaign.status',
+    status: 'campaign_messages.status',
-    first_name: 'subscribers.first_name',
+    first_name: 'subscriptions.first_name',
-    last_name: 'subscribers.last_name',
+    last_name: 'subscriptions.last_name',
-    email: 'subscribers.email'
+    email: 'subscriptions.email'
 };
 
 async function getCampaignResults(context, campaign, select, extra) {
@@ -164,7 +164,7 @@ async function getCampaignResults(context, campaign, select, extra) {
         /* Dropdown and checkbox groups have field.column == null
            TODO - For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
         if (fld.column) {
-            fieldsMapping[fld.key.toLowerCase()] = 'subscribers.' + fld.column;
+            fieldsMapping[fld.key.toLowerCase()] = 'subscriptions.' + fld.column;
         }
     }
 
@@ -180,9 +180,11 @@ async function getCampaignResults(context, campaign, select, extra) {
         }
     }
 
-    let query = knex(`subscription__${campaign.list} AS subscribers`)
+    let query = knex(`subscription__${campaign.list} AS subscriptions`)
-        .innerJoin(`campaign__${campaign.id} AS campaign`, 'subscribers.id', 'campaign.subscription')
+        .innerJoin('campaign_messages', 'subscriptions.id', 'campaign_messages.subscription')
-        .leftJoin(`campaign_tracker__${campaign.id} AS tracker`, 'subscribers.id', 'tracker.subscriber')
+        .leftJoin('campaign_links', 'subscriptions.id', 'campaign_links.subscription')
+        .where('campaign_messages.list', campaign.list)
+        .where('campaign_links.list', campaign.list)
         .select(selFields);
 
     if (extra) {
@@ -194,17 +196,15 @@ async function getCampaignResults(context, campaign, select, extra) {
 
 
 
-module.exports = {
+module.exports.ReportState = ReportState;
-    ReportState,
+module.exports.hash = hash;
-    hash,
+module.exports.getByIdWithTemplate = getByIdWithTemplate;
-    getByIdWithTemplate,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove,
+module.exports.removeAllByReportTemplateIdTx = removeAllByReportTemplateIdTx;
-    removeAllByReportTemplateIdTx,
+module.exports.updateFields = updateFields;
-    updateFields,
+module.exports.listByState = listByState;
-    listByState,
+module.exports.bulkChangeState = bulkChangeState;
-    bulkChangeState,
+module.exports.getCampaignResults = getCampaignResults;
-    getCampaignResults,
-};

models/segments.js

@@ -408,16 +408,14 @@ async function getQueryGeneratorTx(tx, listId, id) {
 } 
 
 // This is to handle circular dependency with fields.js
-Object.assign(module.exports, {
+module.exports.hash = hash;
-    hash,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.listIdName = listIdName;
-    listIdName,
+module.exports.getById = getById;
-    getById,
+module.exports.getByIdTx = getByIdTx;
-    getByIdTx,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove,
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
-    removeAllByListIdTx,
+module.exports.removeRulesByColumnTx = removeRulesByColumnTx;
-    removeRulesByColumnTx,
+module.exports.getQueryGeneratorTx = getQueryGeneratorTx;
-    getQueryGeneratorTx
-});

models/send-configurations.js

@@ -132,14 +132,12 @@ async function getSystemSendConfiguration() {
     return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId(), false);
 }
 
-module.exports = {
+module.exports.MailerType = MailerType;
-    MailerType,
+module.exports.hash = hash;
-    hash,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.getByIdTx = getByIdTx;
-    getByIdTx,
+module.exports.getById = getById;
-    getById,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove,
+module.exports.getSystemSendConfiguration = getSystemSendConfiguration;
-    getSystemSendConfiguration
-};

models/settings.js

@@ -55,9 +55,7 @@ async function set(context, data) {
     // FIXME - recreate mailers, notify senders to recreate the mailers
 }
 
-module.exports = {
+module.exports.hash = hash;
-    hash,
+module.exports.get = get;
-    get,
+module.exports.set = set;
-    set
-};
 

models/shares.js

@@ -614,25 +614,23 @@ async function getPermissionsTx(tx, context, entityTypeId, entityId) {
     return rows.map(x => x.operation);
 }
 
-module.exports = {
+module.exports.listByEntityDTAjax = listByEntityDTAjax;
-    listByEntityDTAjax,
+module.exports.listByUserDTAjax = listByUserDTAjax;
-    listByUserDTAjax,
+module.exports.listUnassignedUsersDTAjax = listUnassignedUsersDTAjax;
-    listUnassignedUsersDTAjax,
+module.exports.listRolesDTAjax = listRolesDTAjax;
-    listRolesDTAjax,
+module.exports.assign = assign;
-    assign,
+module.exports.rebuildPermissionsTx = rebuildPermissionsTx;
-    rebuildPermissionsTx,
+module.exports.rebuildPermissions = rebuildPermissions;
-    rebuildPermissions,
+module.exports.removeDefaultShares = removeDefaultShares;
-    removeDefaultShares,
+module.exports.enforceEntityPermission = enforceEntityPermission;
-    enforceEntityPermission,
+module.exports.enforceEntityPermissionTx = enforceEntityPermissionTx;
-    enforceEntityPermissionTx,
+module.exports.enforceTypePermission = enforceTypePermission;
-    enforceTypePermission,
+module.exports.enforceTypePermissionTx = enforceTypePermissionTx;
-    enforceTypePermissionTx,
+module.exports.checkEntityPermission = checkEntityPermission;
-    checkEntityPermission,
+module.exports.checkTypePermission = checkTypePermission;
-    checkTypePermission,
+module.exports.enforceGlobalPermission = enforceGlobalPermission;
-    enforceGlobalPermission,
+module.exports.checkGlobalPermission = checkGlobalPermission;
-    checkGlobalPermission,
+module.exports.throwPermissionDenied = throwPermissionDenied;
-    throwPermissionDenied,
+module.exports.regenerateRoleNamesTable = regenerateRoleNamesTable;
-    regenerateRoleNamesTable,
+module.exports.getGlobalPermissions = getGlobalPermissions;
-    getGlobalPermissions,
+module.exports.getPermissionsTx = getPermissionsTx;
-    getPermissionsTx
-};

models/subscriptions.js

@@ -13,6 +13,7 @@ const { enforce, filterObject } = require('../lib/helpers');
 const moment = require('moment');
 const { formatDate, formatBirthday } = require('../shared/date');
 const crypto = require('crypto');
+const campaigns = require('./campaigns');
 
 const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']);
 
@@ -79,10 +80,6 @@ function getSubscriptionTableName(listId) {
     return `subscription__${listId}`;
 }
 
-function getCampaignTableName(campaignId) {
-    return `campaign__${campaignId}`;
-}
-
 async function getGroupedFieldsMap(tx, listId) {
     const groupedFields = await fields.listGroupedTx(tx, listId);
     const result = {};
@@ -596,58 +593,54 @@ async function removeByEmailAndGet(context, listId, email) {
 }
 
 
-async function _unsubscribeAndGetTx(tx, context, listId, existingSubscription, campaignCid) {
+async function _changeStatusTx(tx, context, listId, existing, newStatus) {
+    enforce(newStatus !== SubscriptionStatus.SUBSCRIBED);
+
     await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
 
-    if (!(existingSubscription && existingSubscription.status === SubscriptionStatus.SUBSCRIBED)) {
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update({
+        status: newStatus
+    });
+
+    if (existing.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).decrement('subscribers', 1);
+    }
+}
+
+async function _unsubscribeExistingAndGetTx(tx, context, listId, existing) {
+    if (!(existing && existing.status === SubscriptionStatus.SUBSCRIBED)) {
         throw new interoperableErrors.NotFoundError();
     }
 
-    existingSubscription.status = SubscriptionStatus.UNSUBSCRIBED;
+    await _changeStatusTx(tx, context, listId, existing, SubscriptionStatus.UNSUBSCRIBED);
 
-    await tx(getSubscriptionTableName(listId)).where('id', existingSubscription.id).update({
+    existing.status = SubscriptionStatus.SUBSCRIBED;
-        status: SubscriptionStatus.UNSUBSCRIBED
-    });
 
-    await tx('lists').where('id', listId).decrement('subscribers', 1);
+    return existing;
-
-    if (campaignCid) {
-        const campaign = await tx('campaigns').where('cid', campaignCid);
-        const subscriptionInCampaign = await tx(getCampaignTableName(campaign.id)).where({subscription: existingSubscription.id, list: listId});
-
-        if (!subscriptionInCampaign) {
-            throw new Error('Invalid campaign.')
 }
 
-        if (subscriptionInCampaign.status === SubscriptionStatus.SUBSCRIBED) {
-            await tx('campaigns').where('id', campaign.id).increment('unsubscribed', 1);
-            await tx(getCampaignTableName(campaign.id)).where({subscription: existingSubscription.id, list: listId}).update({
-                status: SubscriptionStatus.UNSUBSCRIBED
-            });
-        }
-    }
-
-    return existingSubscription;
-}
-
-
 async function unsubscribeByIdAndGet(context, listId, subscriptionId) {
     return await knex.transaction(async tx => {
         const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
-        return await _unsubscribeAndGetTx(tx, context, listId, existing);
+        return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
     });
 }
 
 async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaignCid) {
     return await knex.transaction(async tx => {
         const existing = await tx(getSubscriptionTableName(listId)).where('cid', subscriptionCid).first();
-        return await _unsubscribeAndGetTx(tx, context, listId, existing, campaignCid);
+
+        if (campaignCid) {
+            await campaigns.changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaignCid, listId, existing.id, SubscriptionStatus.UNSUBSCRIBED);
+        }
+
+        return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
     });
 }
 
 async function unsubscribeByEmailAndGetTx(tx, context, listId, email) {
     const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
-    return await _unsubscribeAndGetTx(tx, context, listId, existing);
+    return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
 }
 
 async function unsubscribeByEmailAndGet(context, listId, email) {
@@ -656,6 +649,13 @@ async function unsubscribeByEmailAndGet(context, listId, email) {
     });
 }
 
+async function changeStatusTx(tx, context, listId, subscriptionId, subscriptionStatus) {
+    const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+    await _changeStatusTx(tx, context, listId, existing, subscriptionStatus);
+}
+
+
+
 async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
@@ -722,25 +722,25 @@ async function getListsWithEmail(context, email) {
     });
 }
 
-module.exports = {
+module.exports.getSubscriptionTableName = getSubscriptionTableName;
-    hashByList,
+module.exports.hashByList = hashByList;
-    getById,
+module.exports.getById = getById;
-    getByCid,
+module.exports.getByCid = getByCid;
-    getByEmail,
+module.exports.getByEmail = getByEmail;
-    list,
+module.exports.list = list;
-    listDTAjax,
+module.exports.listDTAjax = listDTAjax;
-    serverValidate,
+module.exports.serverValidate = serverValidate;
-    create,
+module.exports.create = create;
-    getGroupedFieldsMap,
+module.exports.getGroupedFieldsMap = getGroupedFieldsMap;
-    createTxWithGroupedFieldsMap,
+module.exports.createTxWithGroupedFieldsMap = createTxWithGroupedFieldsMap;
-    updateWithConsistencyCheck,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    remove,
+module.exports.remove = remove;
-    removeByEmailAndGet,
+module.exports.removeByEmailAndGet = removeByEmailAndGet;
-    unsubscribeByCidAndGet,
+module.exports.unsubscribeByCidAndGet = unsubscribeByCidAndGet;
-    unsubscribeByIdAndGet,
+module.exports.unsubscribeByIdAndGet = unsubscribeByIdAndGet;
-    unsubscribeByEmailAndGet,
+module.exports.unsubscribeByEmailAndGet = unsubscribeByEmailAndGet;
-    unsubscribeByEmailAndGetTx,
+module.exports.unsubscribeByEmailAndGetTx = unsubscribeByEmailAndGetTx;
-    updateAddressAndGet,
+module.exports.updateAddressAndGet = updateAddressAndGet;
-    updateManaged,
+module.exports.updateManaged = updateManaged;
-    getListsWithEmail
+module.exports.getListsWithEmail = getListsWithEmail;
-};
+module.exports.changeStatusTx = changeStatusTx;

models/templates.js

@@ -102,12 +102,10 @@ async function remove(context, id) {
     });
 }
 
-module.exports = {
+module.exports.hash = hash;
-    hash,
+module.exports.getByIdTx = getByIdTx;
-    getByIdTx,
+module.exports.getById = getById;
-    getById,
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.remove = remove;
-    remove
-};

models/triggers.js

@@ -137,14 +137,12 @@ async function removeAllByCampaignIdTx(tx, context, campaignId) {
 
 
 // This is to handle circular dependency with campaigns.js
-Object.assign(module.exports, {
+module.exports.hash = hash;
-    hash,
+module.exports.getById = getById;
-    getById,
+module.exports.listByCampaignDTAjax = listByCampaignDTAjax;
-    listByCampaignDTAjax,
+module.exports.listByListDTAjax = listByListDTAjax;
-    listByListDTAjax,
+module.exports.create = create;
-    create,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.removeTx = removeTx;
-    removeTx,
+module.exports.remove = remove;
-    remove,
+module.exports.removeAllByCampaignIdTx = removeAllByCampaignIdTx;
-    removeAllByCampaignIdTx
-});

models/users.js

@@ -418,24 +418,22 @@ async function getByRestrictedAccessToken(token) {
 }
 
 
-module.exports = {
+module.exports.listDTAjax = listDTAjax;
-    listDTAjax,
+module.exports.remove = remove;
-    remove,
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
-    updateWithConsistencyCheck,
+module.exports.create = create;
-    create,
+module.exports.hash = hash;
-    hash,
+module.exports.getById = getById;
-    getById,
+module.exports.serverValidate = serverValidate;
-    serverValidate,
+module.exports.getByAccessToken = getByAccessToken;
-    getByAccessToken,
+module.exports.getByUsername = getByUsername;
-    getByUsername,
+module.exports.getByUsernameIfPasswordMatch = getByUsernameIfPasswordMatch;
-    getByUsernameIfPasswordMatch,
+module.exports.getAccessToken = getAccessToken;
-    getAccessToken,
+module.exports.resetAccessToken = resetAccessToken;
-    resetAccessToken,
+module.exports.sendPasswordReset = sendPasswordReset;
-    sendPasswordReset,
+module.exports.isPasswordResetTokenValid = isPasswordResetTokenValid;
-    isPasswordResetTokenValid,
+module.exports.resetPassword = resetPassword;
-    resetPassword,
+module.exports.getByRestrictedAccessToken = getByRestrictedAccessToken;
-    getByRestrictedAccessToken,
+module.exports.getRestrictedAccessToken = getRestrictedAccessToken;
-    getRestrictedAccessToken,
+module.exports.refreshRestrictedAccessToken = refreshRestrictedAccessToken;
-    refreshRestrictedAccessToken,
+module.exports.registerRestrictedAccessTokenMethod = registerRestrictedAccessTokenMethod;
-    registerRestrictedAccessTokenMethod
-};

obsolete/services/verp-server.js

@@ -3,35 +3,19 @@
 // FIXME - port for the new campaigns model
 
 const { nodeifyFunction } = require('../lib/nodeify');
-const getSettings = nodeifyFunction(require('../models/settings').get);
+const log = require('npmlog');
+const config = require('config');
+const verpHelpers = require('../lib/verp-helpers');
+const campaigns = require('../models/campaigns');
+const BounceHandler = require('bounce-handler').BounceHandler;
+const SMTPServer = require('smtp-server').SMTPServer;
 
-let log = require('npmlog');
+async function onRcptTo(address, session) {
-let config = require('config');
-let campaigns = require('../lib/models/campaigns');
-let BounceHandler = require('bounce-handler').BounceHandler;
-let SMTPServer = require('smtp-server').SMTPServer;
 
-// Setup server
-let server = new SMTPServer({
 
-    // log to console
+    const user = address.address.split('@').shift();
-    logger: false,
+    const host = address.address.split('@').pop();
 
-    banner: 'Mailtrain VERP bouncer',
-
-    disabledCommands: ['AUTH', 'STARTTLS'],
-
-    onRcptTo: (address, session, callback) => {
-
-        getSettings(['verpHostname'], (err, configItems) => {
-            if (err) {
-                err = new Error('Failed to load configuration');
-                err.responseCode = 421;
-                return callback(err);
-            }
-
-            let user = address.address.split('@').shift();
-            let host = address.address.split('@').pop();
 
         if (host !== configItems.verpHostname || !/^[a-z0-9_-]+\.[a-z0-9_-]+\.[a-z0-9_-]+$/i.test(user)) {
             err = new Error('Unknown user ' + address.address);
@@ -60,10 +44,9 @@ let server = new SMTPServer({
             callback();
         });
     });
-    },
+}
 
-    // Handle message stream
+async function onData(stream, session) {
-    onData: (stream, session, callback) => {
     let chunks = [];
     let chunklen = 0;
     stream.on('data', chunk => {
@@ -100,7 +83,21 @@ let server = new SMTPServer({
             });
         }
     });
+
 }
+
+// Setup server
+const server = new SMTPServer({
+
+    // log to console
+    logger: false,
+
+    banner: 'Mailtrain VERP bouncer',
+
+    disabledCommands: ['AUTH', 'STARTTLS'],
+
+    onRcptTo: nodeifyFunction(onRcptTo),
+    onData: nodeifyFunction(onData)
 });
 
 module.exports = callback => {
@@ -135,7 +132,7 @@ module.exports = callback => {
 
     let hosts;
     if (typeof config.verp.host === 'string' && config.verp.host) {
-        hosts = config.verp.host.trim().split(',').map(host => host.trim()).filter(host => host.trim());
+        hosts = config.verp.host.trim().split(',').map(host => host.trim()).filter(host => !!host);
         if (hosts.indexOf('*') >= 0 || hosts.indexOf('all') >= 0) {
             hosts = [false];
         }
@@ -144,7 +141,7 @@ module.exports = callback => {
     }
 
     let pos = 0;
-    let startNextHost = () => {
+    const startNextHost = () => {
         if (pos >= hosts.length) {
             started = true;
             return setImmediate(callback);

routes/rest/campaigns.js

@@ -18,6 +18,9 @@ router.postAsync('/campaigns-others-by-list-table/:campaignId/:listIds', passpor
     return res.json(await campaigns.listOthersWhoseListsAreIncludedDTAjax(req.context, req.params.campaignId, req.params.listIds.split(';'), req.body));
 });
 
+router.postAsync('/campaigns-test-users-table/:campaignId', passport.loggedIn, async (req, res) => {
+    return res.json(await campaigns.listTestUsersDTAjax(req.context, req.params.campaignId, req.body));
+});
 
 router.getAsync('/campaigns-settings/:campaignId', passport.loggedIn, async (req, res) => {
     const campaign = await campaigns.getById(req.context, req.params.campaignId, true, campaigns.Content.WITHOUT_SOURCE_CUSTOM);
@@ -25,6 +28,11 @@ router.getAsync('/campaigns-settings/:campaignId', passport.loggedIn, async (req
     return res.json(campaign);
 });
 
+router.getAsync('/campaigns-stats/:campaignId', passport.loggedIn, async (req, res) => {
+    const campaign = await campaigns.getById(req.context, req.params.campaignId, true, campaigns.Content.SETTINGS_WITH_STATS);
+    return res.json(campaign);
+});
+
 router.getAsync('/campaigns-content/:campaignId', passport.loggedIn, async (req, res) => {
     const campaign = await campaigns.getById(req.context, req.params.campaignId, true, campaigns.Content.ONLY_SOURCE_CUSTOM);
     campaign.hash = campaigns.hash(campaign, campaigns.Content.ONLY_SOURCE_CUSTOM);
@@ -56,5 +64,21 @@ router.deleteAsync('/campaigns/:campaignId', passport.loggedIn, passport.csrfPro
     return res.json();
 });
 
+router.postAsync('/campaign-start/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.start(req.context, req.params.campaignId, null));
+});
+
+router.postAsync('/campaign-start-at/:campaignId/:dateTime', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.start(req.context, req.params.campaignId, new Date(req.params.dateTime)));
+});
+
+
+router.postAsync('/campaign-stop/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.stop(req.context, req.params.campaignId));
+});
+
+router.postAsync('/campaign-reset/:campaignId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await campaigns.reset(req.context, req.params.campaignId));
+});
 
 module.exports = router;

routes/rest/lists.js

@@ -10,6 +10,10 @@ router.postAsync('/lists-table', passport.loggedIn, async (req, res) => {
     return res.json(await lists.listDTAjax(req.context, req.body));
 });
 
+router.postAsync('/lists-with-segment-by-campaign-table/:campaignId', passport.loggedIn, async (req, res) => {
+    return res.json(await lists.listWithSegmentByCampaignDTAjax(req.context, req.params.campaignId, req.body));
+});
+
 router.getAsync('/lists/:listId', passport.loggedIn, async (req, res) => {
     const list = await lists.getByIdWithListFields(req.context, req.params.listId);
     list.hash = lists.hash(list);

services/sender-master.js

@@ -5,35 +5,187 @@ const fork = require('child_process').fork;
 const log = require('npmlog');
 const path = require('path');
 const knex = require('../lib/knex');
+const {CampaignStatus, CampaignType} = require('../shared/campaigns');
+const { enforce } = require('../lib/helpers');
+const campaigns = require('../models/campaigns');
+const subscriptions = require('../models/subscriptions');
+const { SubscriptionStatus } = require('../shared/lists');
+const segments = require('../models/segments');
 
 let messageTid = 0;
-let workerProcesses = new Map();
+const workerProcesses = new Map();
 
-let running = false;
+const idleWorkers = [];
+
+let campaignSchedulerRunning = false;
+let workerSchedulerRunning = false;
+
+const campaignsCheckPeriod = 5 * 1000;
+const retrieveBatchSize = 1000;
+const workerBatchSize = 100;
+
+const messageQueue = new Map(); // campaignId -> [{listId, email}]
+const messageQueueCont = new Map(); // campaignId -> next batch callback
+
+const workerSchedulerCont = null;
+
+
+function messagesProcessed(workerId) {
+    idleWorkers.push(workerId);
+
+    if (workerSchedulerCont) {
+        const cont = workerSchedulerCont;
+        setImmediate(workerSchedulerCont);
+        workerSchedulerCont = null;
+    }
+}
+
+async function scheduleWorkers() {
+    async function getAvailableWorker() {
+        if (idleWorkers.length > 0) {
+            return idleWorkers.shift();
+
+        } else {
+            const workerAvailable = new Promise(resolve => {
+                workerSchedulerCont = resolve;
+            });
+
+            await workerAvailable;
+            return idleWorkers.shift();
+        }
+    }
+
+
+    if (workerSchedulerRunning) {
+        return;
+    }
+
+    workerSchedulerRunning = true;
+    let workerId = await getAvailableWorker();
+
+    let keepLooping = true;
+
+    while (keepLooping) {
+        keepLooping = false;
+
+        for (const campaignId of messageQueue.keys()) {
+            const queue = messageQueue.get(campaignId);
+
+            if (queue.length > 0) {
+                const msgs = queue.splice(0, workerBatchSize);
+
+                if (queue.length === 0 && messageQueueCont.has(campaignId)) {
+                    const scheduleMessages = messageQueueCont.get(campaignId);
+                    setImmediate(scheduleMessages);
+                }
+
+                sendToWorker(workerId, 'process-messages', msgs);
+                workerId = await getAvailableWorker();
+
+                keepLooping = true;
+            }
+        }
+    }
+
+    idleWorkers.push(workerId);
+
+    workerSchedulerRunning = false;
+}
 
-/*
-const path = require('path');
-const log = require('npmlog');
-const fsExtra = require('fs-extra-promise');
-const {ImportSource, MappingType, ImportStatus, RunStatus} = require('../shared/imports');
-const imports = require('../models/imports');
-const fields = require('../models/fields');
-const subscriptions = require('../models/subscriptions');
-const { Writable } = require('stream');
-const { cleanupFromPost, enforce } = require('../lib/helpers');
-const contextHelpers = require('../lib/context-helpers');
-const tools = require('../lib/tools');
-const shares = require('../models/shares');
-const _ = require('../lib/translate')._;
-*/
 
 
 async function processCampaign(campaignId) {
-    const campaignSubscribersTable = 'campaign__' + campaignId;
+    async function finish() {
-
+        await knex('campaigns').where('id', campaignId).update({status: CampaignStatus.FINISHED});
-
+        messageQueue.delete(campaignId);
     }
 
+    const msgQueue = [];
+    messageQueue.set(campaignId, msgQueue);
+
+    while (true) {
+        const cpg = await knex('campaigns').where('id', campaignId).first();
+
+        if (cpg.status === CampaignStatus.PAUSED) {
+            await finish();
+            return;
+        }
+
+        let qryGen;
+        await knex.transaction(async tx => {
+            qryGen = await campaigns.getSubscribersQueryGeneratorTx(tx, campaignId, true, retrieveBatchSize);
+        });
+
+        if (qryGen) {
+            const qry = qryGen(knex).select(['pending_subscriptions.email', 'campaign_lists.list']);
+            const subs = await qry;
+
+            if (subs.length === 0) {
+                await finish();
+                return;
+            }
+
+            for (const sub of subs) {
+                msgQueue.push({
+                    listId: sub.list,
+                    email: sub.email
+                });
+            }
+
+            const nextBatchNeeded = new Promise(resolve => {
+                messageQueueCont.set(campaignId, resolve);
+            });
+
+            // noinspection JSIgnoredPromiseFromCall
+            setImmediate(scheduleWorkers);
+
+            await nextBatchNeeded;
+
+        } else {
+            await finish();
+            return;
+        }
+    }
+}
+
+
+async function scheduleCampaigns() {
+    if (campaignSchedulerRunning) {
+        return;
+    }
+
+    campaignSchedulerRunning = true;
+
+    while (true) {
+        let campaignId = 0;
+
+        await knex.transaction(async tx => {
+            const scheduledCampaign = await tx('campaigns')
+                .whereIn('campaigns.type', [CampaignType.REGULAR, CampaignType.RSS_ENTRY])
+                .where('campaigns.status', CampaignStatus.SCHEDULED)
+                .where(qry => qry.whereNull('campaigns.scheduled').orWhere('campaigns.scheduled', '<=', new Date()))
+                .select(['id'])
+                .first();
+
+            if (scheduledCampaign) {
+                await tx('campaigns').where('id', scheduledCampaign.id).update({status: CampaignStatus.SENDING});
+                campaignId = scheduledCampaign.id;
+            }
+        });
+
+        if (campaignId) {
+            // noinspection JSIgnoredPromiseFromCall
+            processCampaign(campaignId);
+
+        } else {
+            break;
+        }
+    }
+
+    campaignSchedulerRunning = false;
+}
+
+
 async function spawnWorker(workerId) {
     return await new Promise((resolve, reject) => {
         log.verbose('Senders', `Spawning worker process ${workerId}`);
@@ -48,7 +200,11 @@ async function spawnWorker(workerId) {
                 if (msg.type === 'worker-started') {
                     log.info('Senders', `Worker process ${workerId} started`);
                     return resolve();
+
+                } else if (msg.type === 'messages-processed') {
+                    messageProcessed(workerId);
                 }
+
             }
         });
 
@@ -57,21 +213,10 @@ async function spawnWorker(workerId) {
         });
 
         workerProcesses.set(workerId, senderProcess);
+        idleWorkers.push(workerId);
     });
 }
 
-async function run() {
-    if (running) {
-        return;
-    }
-
-    running = true;
-
-    // FIXME
-
-    running = false;
-}
-
 function sendToWorker(workerId, msgType, data) {
     workerProcesses.get(workerId).send({
         type: msgType,
@@ -82,6 +227,14 @@ function sendToWorker(workerId, msgType, data) {
     messageTid++;
 }
 
+
+function periodicCampaignsCheck() {
+    // noinspection JSIgnoredPromiseFromCall
+    scheduleCampaigns();
+
+    setTimeout(periodicCampaignsCheck, campaignsCheckPeriod);
+}
+
 async function init() {
     const spawnWorkerFutures = [];
     let workerId;
@@ -95,12 +248,13 @@ async function init() {
         if (msg) {
             const type = msg.type;
 
-            if (type === 'scheduleCheck') {
+            if (type === 'schedule-check') {
-                // FIXME
+                // noinspection JSIgnoredPromiseFromCall
+                scheduleCampaigns();
 
-            } else if (type === 'reloadConfig') {
+            } else if (type === 'reload-config') {
                 for (const worker of workerProcesses.keys()) {
-                    sendToWorker(workerId, 'reloadConfig', msg.data);
+                    sendToWorker(workerId, 'reload-config', msg.data);
                 }
             }
         }
@@ -110,7 +264,7 @@ async function init() {
         type: 'sender-started'
     });
 
-    run();
+    periodicCampaignsCheck();
 }
 
 init();

services/sender-worker.js

@@ -23,7 +23,7 @@ const shares = require('../models/shares');
 const _ = require('../lib/translate')._;
 */
 
-async function sendMail() {
+async function processMessages(msgs) {
     if (running) {
         log.error('Senders', `Worker ${workerId} assigned work while working`);
         return;
@@ -31,9 +31,12 @@ async function sendMail() {
 
     running = true;
 
+    console.log(msgs);
     // FIXME
 
     running = false;
+
+    sendToMaster('messages-processed');
 }
 
 function sendToMaster(msgType) {
@@ -46,11 +49,12 @@ process.on('message', msg => {
     if (msg) {
         const type = msg.type;
 
-        if (type === 'reloadConfig') {
+        if (type === 'reload-config') {
             mailers.invalidateMailer(msg.data.sendConfigurationId);
 
-        } else if (type === 'sendMail') {
+        } else if (type === 'process-messages') {
-            // FIXME
+            // noinspection JSIgnoredPromiseFromCall
+            processMessages(msg.data)
         }
 
     }

services/verp-server.js

@@ -0,0 +1,145 @@
+'use strict';
+
+const { nodeifyFunction, nodeifyPromise } = require('../lib/nodeify');
+const log = require('npmlog');
+const config = require('config');
+const {MailerError} = require('../lib/mailers');
+const campaigns = require('../models/campaigns');
+const contextHelpers = require('../lib/context-helpers');
+const {SubscriptionStatus} = require('../shared/lists');
+
+const BounceHandler = require('bounce-handler').BounceHandler;
+const SMTPServer = require('smtp-server').SMTPServer;
+
+async function onRcptTo(address, session) {
+    const addrSplit = address.split('@');
+
+    if (addrSplit.length !== 2) {
+        throw new MailerError('Unknown user ' + address.address, 510);
+    }
+
+    const [user, host] = addrSplit;
+
+    const message = await campaigns.getMessageByCid(user);
+
+    if (!message) {
+        throw new MailerError('Unknown user ' + address.address, 510);
+    }
+
+    if (message.verp_hostname !== host) {
+        throw new MailerError('Unknown user ' + address.address, 510);
+    }
+
+    session.message = message;
+
+    log.verbose('VERP', 'Incoming message for Campaign %s, List %s, Subscription %s', cids.campaignId, cids.listId, cids.subscriptionId);
+}
+
+function onData(stream, session, callback) {
+    let chunks = [];
+    let totalLen = 0;
+
+    stream.on('data', chunk => {
+        if (!chunk || !chunk.length || totalLen > 60 * 1024) {
+            return;
+        }
+        chunks.push(chunk);
+        totalLen += chunk.length;
+    });
+
+    stream.on('end', () => nodeifyPromise(onStreamEnd(), callback));
+
+    const onStreamEnd = async () => {
+        const body = Buffer.concat(chunks, totalLen).toString();
+
+        const bh = new BounceHandler();
+        let bounceResult;
+
+        try {
+            bounceResult = [].concat(bh.parse_email(body) || []).shift();
+        } catch (E) {
+            log.error('Bounce', 'Failed parsing bounce message');
+            log.error('Bounce', JSON.stringify(body));
+        }
+
+        if (!bounceResult || ['failed', 'transient'].indexOf(bounceResult.action) < 0) {
+            return 'Message accepted';
+        } else {
+            await campaigns.changeStatusByMessage(contextHelpers.getAdminContext(), session.message, SubscriptionStatus.BOUNCED, bounceResult.action === 'failed');
+            log.verbose('VERP', 'Marked message %s as unsubscribed', session.message.campaign);
+        }
+    };
+}
+
+// Setup server
+const server = new SMTPServer({
+
+    // log to console
+    logger: false,
+
+    banner: 'Mailtrain VERP bouncer',
+
+    disabledCommands: ['AUTH', 'STARTTLS'],
+
+    onRcptTo: nodeifyFunction(onRcptTo),
+    onData: onData
+});
+
+module.exports = callback => {
+    if (!config.verp.enabled) {
+        return setImmediate(callback);
+    }
+
+    let started = false;
+
+    server.on('error', err => {
+        const port = config.verp.port;
+        const bind = typeof port === 'string' ? 'Pipe ' + port : 'Port ' + port;
+
+        switch (err.code) {
+            case 'EACCES':
+                log.error('VERP', '%s requires elevated privileges', bind);
+                break;
+            case 'EADDRINUSE':
+                log.error('VERP', '%s is already in use', bind);
+                break;
+            case 'ECONNRESET': // Usually happens when a client does not disconnect cleanly
+            case 'EPIPE': // Remote connection was closed before the server attempted to send data
+            default:
+                log.error('VERP', err);
+        }
+
+        if (!started) {
+            started = true;
+            return callback(err);
+        }
+    });
+
+    let hosts;
+    if (typeof config.verp.host === 'string' && config.verp.host) {
+        hosts = config.verp.host.trim().split(',').map(host => host.trim()).filter(host => !!host);
+        if (hosts.indexOf('*') >= 0 || hosts.indexOf('all') >= 0) {
+            hosts = [false];
+        }
+    } else {
+        hosts = [false];
+    }
+
+    let pos = 0;
+    const startNextHost = () => {
+        if (pos >= hosts.length) {
+            started = true;
+            return setImmediate(callback);
+        }
+        let host = hosts[pos++];
+        server.listen(config.verp.port, host, () => {
+            if (started) {
+                return server.close();
+            }
+            log.info('VERP', 'Server listening on %s:%s', host || '*', config.verp.port);
+            setImmediate(startNextHost);
+        });
+    };
+
+    startNextHost();
+};

setup/knex/migrations/20170506102634_v1_to_v2.js

@@ -72,14 +72,14 @@ async function migrateBase(knex) {
             });
     */
 
-    // Original Mailtrain migration is executed before this one. So here we check that the original migration
+    // The original Mailtrain migration is executed before this one. So here we check whether the original migration
-    // ended where it should have and we take it from here.
+    // ended where it should have and we take it from there.
     const row = await knex('settings').where({key: 'db_schema_version'}).first('value');
     if (!row || Number(row.value) !== 33) {
         throw new Error('Unsupported DB schema version: ' + row.value);
     }
 
-    // We have to update data types of primary keys and related foreign keys. Mailtrain uses unsigned int(11), while
+    // Update data types of primary keys and related foreign keys. Mailtrain uses unsigned int(11), while
     // Knex uses unsigned int (which is unsigned int(10) ).
     await knex.schema
         .raw('ALTER TABLE `attachments` MODIFY `id` int unsigned not null auto_increment')
@@ -894,6 +894,40 @@ async function migrateCampaigns(knex) {
         table.integer('segment').unsigned().references('segments.id').onDelete('CASCADE');
     });
 
+    await knex.schema.raw('CREATE TABLE `campaign_messages` (\n' +
+        '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
+        '  `campaign` int(10) unsigned NOT NULL,\n' +
+        '  `list` int(10) unsigned NOT NULL,\n' +
+        '  `subscription` int(10) unsigned NOT NULL,\n' +
+        '  `send_configuration` int(10) unsigned NOT NULL,\n' +
+        '  `status` tinyint(4) unsigned NOT NULL DEFAULT \'0\',\n' +
+        '  `response` varchar(255) DEFAULT NULL,\n' +
+        '  `response_id` varchar(255) CHARACTER SET ascii DEFAULT NULL,\n' +
+        '  `updated` timestamp NULL DEFAULT NULL,\n' +
+        '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
+        '  PRIMARY KEY (`id`),\n' +
+        '  UNIQUE KEY `cls` (`campaign`, `list`, `subscription`),\n' +
+        '  KEY `created` (`created`),\n' +
+        '  KEY `response_id` (`response_id`),\n' +
+        '  KEY `status_index` (`status`),\n' +
+        '  KEY `subscription_index` (`subscription`)\n' +
+        ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+    await knex.schema.raw('CREATE TABLE `campaign_links` (\n' +
+        '  `campaign` int(10) unsigned NOT NULL,\n' +
+        '  `list` int(10) unsigned NOT NULL,\n' +
+        '  `subscription` int(10) unsigned NOT NULL,\n' +
+        '  `link` int(10) NOT NULL,\n' +
+        '  `ip` varchar(100) CHARACTER SET ascii DEFAULT NULL,\n' +
+        '  `device_type` varchar(50) DEFAULT NULL,\n' +
+        '  `country` varchar(2) CHARACTER SET ascii DEFAULT NULL,\n' +
+        '  `count` int(11) unsigned NOT NULL DEFAULT \'1\',\n' +
+        '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
+        '  PRIMARY KEY (`campaign`, `list`,`subscription`,`link`),\n' +
+        '  KEY `created_index` (`created`)\n' +
+        ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+
     await knex.schema.table('campaigns', table => {
         table.text('data', 'longtext');
         table.integer('source').unsigned().notNullable();
@@ -907,6 +941,15 @@ async function migrateCampaigns(knex) {
     for (const campaign of campaigns) {
         const data = {};
 
+        await knex.raw('INSERT INTO `campaign_messages` (`id`, `campaign`, `list`, `subscription`, `send_configuration`, `status`, `response`, `response_id`, `updated`, `created`) ' +
+            'SELECT `id`, ' + campaign.id + ', `list`, `subscription`, ' + getSystemSendConfigurationId() + ', `status`, `response`, `response_id`, `updated`, `created` FROM `campaign__' + campaign.id + '`;');
+
+        await knex.raw('INSERT INTO `campaign_links` (`campaign`, `list`, `subscription`, `link`, `ip`, `device_type`, `country`, `count`, `created`) ' +
+            'SELECT ' + campaign.id + ', `list`, `subscriber`, `link`, `ip`, `device_type`, `country`, `count`, `created` FROM `campaign_tracker__' + campaign.id + '`;');
+
+        await knex.schema.dropTableIfExists('campaign__' + campaign.id);
+        await knex.schema.dropTableIfExists('campaign_tracker__' + campaign.id);
+
         if (campaign.type === CampaignType.REGULAR || campaign.type === CampaignType.RSS || campaign.type === CampaignType.RSS_ENTRY || campaign.type === CampaignType.TRIGGERED) {
             if (campaign.template) {
                 let editorType = campaign.editor_name;
@@ -980,6 +1023,7 @@ async function migrateCampaigns(knex) {
         table.integer('send_configuration').unsigned().notNullable().alter();
     });
 
+
     await knex.schema.dropTableIfExists('campaign');
     await knex.schema.dropTableIfExists('campaign_tracker');
 }

shared/campaigns.js

@@ -36,7 +36,10 @@ const CampaignStatus = {
     INACTIVE: 5,
     ACTIVE: 6,
 
-    MAA: 6
+    // For campaign types: NORMAL, RSS_ENTRY
+    SENDING: 7,
+
+    MAX: 8
 };
 
 const campaignOverridables = ['from_name', 'from_email', 'reply_to', 'subject'];