- Refactoring of the mail sending part. Mail queue (table 'queued') is now used also for all test emails.

- More options how to send test emails.
- Fixed problems with pausing a campaign (#593)
- Started rework of transactional sender of templates (#606), however this contains functionality regression at the moment because it does not interpret templates as HBS. It needs HBS option for templates as described in https://github.com/Mailtrain-org/mailtrain/issues/611#issuecomment-502345227

TODO:
- detect sending errors connected to not able to contact the mailer and pause/retry campaing and queued sending - don't mark the recipients as BOUNCED
- add FAILED campaign state and fall into it if sending to campaign consistently fails (i.e. the error with sending is not temporary)
- if the same happends for queued email, delete the message

client/src/account/API.js

@@ -138,7 +138,7 @@ export default class API extends Component {
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/subscribe/B16uVTdW?access_token=${accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/subscribe/B16uVTdW?access_token=${accessToken}`)}' \<br/>
 --data 'EMAIL=test@example.com&amp;MERGE_CHECKBOX=yes&amp;REQUIRE_CONFIRMATION=yes'</pre>
 
                 <h4>POST /api/unsubscribe/:listId – {t('removeSubscription')}</h4>
@@ -165,7 +165,7 @@ export default class API extends Component {
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/unsubscribe/B16uVTdW?access_token=${accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/unsubscribe/B16uVTdW?access_token=${accessToken}`)}' \<br/>
 --data 'EMAIL=test@example.com'</pre>
 
                 <h4>POST /api/delete/:listId – {t('deleteSubscription')}</h4>
@@ -192,7 +192,7 @@ export default class API extends Component {
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/delete/B16uVTdW?access_token=${accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/delete/B16uVTdW?access_token=${accessToken}`)}' \<br/>
 --data 'EMAIL=test@example.com'</pre>
 
                 <h4>POST /api/field/:listId – {t('addNewCustomField')}</h4>
@@ -240,7 +240,7 @@ export default class API extends Component {
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/field/B16uVTdW?access_token=${accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/field/B16uVTdW?access_token=${accessToken}`)}' \<br/>
 --data 'NAME=Birthday&amp;TYPE=birthday-us&amp;VISIBLE=yes'</pre>
 
                 <h4>GET /api/blacklist/get – {t('getListOfBlacklistedEmails')}</h4>
@@ -292,7 +292,7 @@ export default class API extends Component {
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/blacklist/add?access_token={accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/blacklist/add?access_token={accessToken}`)}' \<br/>
 --data 'EMAIL=test@example.com&amp;'</pre>
 
                 <h4>POST /api/blacklist/delete – {t('deleteEmailFromBlacklist')}</h4>
@@ -319,7 +319,7 @@ export default class API extends Component {
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/blacklist/delete?access_token=${accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/blacklist/delete?access_token=${accessToken}`)}' \<br/>
 --data 'EMAIL=test@example.com&amp;'</pre>
 
                 <h4>GET /api/lists/:email – {t('getTheListsAUserHasSubscribedTo')}</h4>
@@ -381,15 +381,14 @@ export default class API extends Component {
                     <li><strong>EMAIL</strong> – {t('emailAddress')} (<em>{t('required')}</em>)</li>
                     <li><strong>SEND_CONFIGURATION_ID</strong> – {t('idOfConfigurationUsedToCreateMailer')}</li>
                     <li><strong>SUBJECT</strong> – {t('subject')}</li>
-                    <li><strong>DATA</strong> – {t('dataPassedToTemplateWhenCompilingWith')}: <em>{'{'} "any": ["type", {'{'}"of": "data"{'}'}] {'}'}</em></li>
+                    <li><strong>VARIABLES</strong> – {t('mapOfTemplatesubjectVariablesToReplace')}</li>
-                    <li><strong>VARIABLES</strong> – {t('mapOfTemplatesubjectVariablesToReplace')}: <em>{'{'} "FOO": "bar" {'}'}</em></li>
                 </ul>
 
                 <p>
                     <strong>{t('example')}</strong>
                 </p>
 
-                <pre>curl -XPOST '{getUrl(`api/templates/1/send?access_token={accessToken}`)}' \
+                <pre>curl -XPOST '{getUrl(`api/templates/1/send?access_token=${accessToken}`)}' \<br/>
 --data 'EMAIL=test@example.com&amp;SUBJECT=Test&amp;VARIABLES[FOO]=bar&amp;VARIABLES[TEST]=example'</pre>
             </div>
         );

client/src/campaigns/CUD.js

@@ -244,7 +244,7 @@ export default class CUD extends Component {
 
         return filterData(data, [
             'name', 'description', 'segment', 'namespace', 'send_configuration',
-            'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override',
+            'from_name_override', 'from_email_override', 'reply_to_override',
             'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url',
             'type', 'source', 'parent', 'lists'
         ]);
@@ -284,6 +284,8 @@ export default class CUD extends Component {
                 send_configuration: null,
                 namespace: mailtrainConfig.user.namespace,
 
+                subject: '',
+
                 click_tracking_disabled: false,
                 open_tracking_disabled: false,
 
@@ -326,6 +328,10 @@ export default class CUD extends Component {
             state.setIn(['name', 'error'], t('nameMustNotBeEmpty'));
         }
 
+        if (!state.getIn(['subject', 'value'])) {
+            state.setIn(['subject', 'error'], t('"Subject" line must not be empty"'));
+        }
+
         if (!state.getIn(['send_configuration', 'value'])) {
             state.setIn(['send_configuration', 'error'], t('sendConfigurationMustBeSelected'));
         }
@@ -592,7 +598,6 @@ export default class CUD extends Component {
             { data: 2, title: t('id'), render: data => <code>{data}</code> },
             { data: 3, title: t('description') },
             { data: 4, title: t('type'), render: data => this.mailerTypes[data].typeName },
-            { data: 5, title: t('created'), render: data => moment(data).fromNow() },
             { data: 6, title: t('namespace') }
         ];
 
@@ -604,10 +609,10 @@ export default class CUD extends Component {
                 const addOverridable = (id, label) => {
                     if(this.state.sendConfiguration[id + '_overridable']){
                         if (this.getFormValue(id + '_overriden')) {
-                            sendSettings.push(<InputField label={t(label)} key={id + '_override'} id={id + '_override'}/>);
+                            sendSettings.push(<InputField label={label} key={id + '_override'} id={id + '_override'}/>);
                         } else {
                             sendSettings.push(
-                                <StaticField key={id + '_original'} label={t(label)} id={id + '_original'} className={styles.formDisabled}>
+                                <StaticField key={id + '_original'} label={label} id={id + '_original'} className={styles.formDisabled}>
                                     {this.state.sendConfiguration[id]}
                                 </StaticField>
                             );
@@ -616,7 +621,7 @@ export default class CUD extends Component {
                     }
                     else{
                         sendSettings.push(
-                            <StaticField key={id + '_original'} label={t(label)} id={id + '_original'} className={styles.formDisabled}>
+                            <StaticField key={id + '_original'} label={label} id={id + '_original'} className={styles.formDisabled}>
                                 {this.state.sendConfiguration[id]}
                             </StaticField>
                         );
@@ -626,7 +631,8 @@ export default class CUD extends Component {
                 addOverridable('from_name', t('fromName'));
                 addOverridable('from_email', t('fromEmailAddress'));
                 addOverridable('reply_to', t('replytoEmailAddress'));
-                addOverridable('subject', t('subjectLine'));
+
+                sendSettings.push(<InputField label={t('subjectLine')} key="subject" id="subject"/>);
             } else {
                 sendSettings =  <AlignedRow>{t('loadingSendConfiguration')}</AlignedRow>
             }
@@ -760,8 +766,6 @@ export default class CUD extends Component {
                     </>
                     }
 
-
-
                     {templateEdit}
 
                     <ButtonRow>

client/src/campaigns/Content.js

@@ -20,7 +20,7 @@ import {getEditForm, getTemplateTypes, getTypeForm, ResourceType} from '../templ
 import axios from '../lib/axios';
 import styles from "../lib/styles.scss";
 import {getUrl} from "../lib/urls";
-import {TestSendModalDialog} from "./TestSendModalDialog";
+import {TestSendModalDialog, TestSendModalDialogMode} from "./TestSendModalDialog";
 import {withComponentMixins} from "../lib/decorator-helpers";
 import {ContentModalDialog} from "../lib/modals";
 
@@ -234,10 +234,11 @@ export default class CustomContent extends Component {
         return (
             <div className={this.state.elementInFullscreen ? styles.withElementInFullscreen : ''}>
                 <TestSendModalDialog
+                    mode={TestSendModalDialogMode.CAMPAIGN_CONTENT}
                     visible={this.state.showTestSendModal}
                     onHide={() => this.setState({showTestSendModal: false})}
                     getDataAsync={this.sendModalGetDataHandler}
-                    entity={this.props.entity}
+                    campaign={this.props.entity}
                 />
                 <ContentModalDialog
                     title={this.state.exportModalTitle}
@@ -261,7 +262,7 @@ export default class CustomContent extends Component {
                         <Button type="submit" className="btn-primary" icon="check" label={t('save')}/>
                         <Button type="submit" className="btn-primary" icon="check" label={t('saveAndLeave')} onClickAsync={async () => await this.submitHandler(CustomContent.AfterSubmitAction.LEAVE)}/>
                         <Button type="submit" className="btn-primary" icon="check" label={t('saveAndGoToStatus')} onClickAsync={async () => await this.submitHandler(CustomContent.AfterSubmitAction.STATUS)}/>
-                        <Button className="btn-success" icon="at" label={t('testSend')} onClickAsync={async () => this.setState({showTestSendModal: true})}/>
+                        <Button className="btn-success" icon="at" label={t('Test send')} onClickAsync={async () => this.setState({showTestSendModal: true})}/>
                     </ButtonRow>
                 </Form>
             </div>

client/src/campaigns/Status.js

@@ -16,6 +16,7 @@ import {CampaignStatus, CampaignType} from "../../../shared/campaigns";
 import moment from 'moment';
 import campaignsStyles from "./styles.scss";
 import {withComponentMixins} from "../lib/decorator-helpers";
+import {TestSendModalDialog, TestSendModalDialogMode} from "./TestSendModalDialog";
 
 
 @withComponentMixins([
@@ -25,7 +26,7 @@ import {withComponentMixins} from "../lib/decorator-helpers";
     withPageHelpers,
     requiresAuthenticatedUser
 ])
-class TestUser extends Component {
+class PreviewForTestUserModalDialog extends Component {
     constructor(props) {
         super(props);
         this.initForm({
@@ -34,7 +35,9 @@ class TestUser extends Component {
     }
 
     static propTypes = {
-        entity: PropTypes.object.isRequired
+        visible: PropTypes.bool.isRequired,
+        onHide: PropTypes.func.isRequired,
+        entity: PropTypes.object.isRequired,
     }
 
     localValidateFormValues(state) {
@@ -64,6 +67,10 @@ class TestUser extends Component {
         }
     }
 
+    async hideModal() {
+        this.props.onHide();
+    }
+
     render() {
         const t = this.props.t;
 
@@ -76,12 +83,14 @@ class TestUser extends Component {
         ];
 
         return (
+            <ModalDialog hidden={!this.props.visible} title={t('Preview Campaign')} onCloseAsync={() => this.hideModal()} buttons={[
+                { label: t('preview'), className: 'btn-primary', onClickAsync: ::this.previewAsync },
+                { label: t('close'), className: 'btn-danger', onClickAsync: ::this.hideModal }
+            ]}>
                 <Form stateOwner={this}>
-                <TableSelect id="testUser" label={t('previewCampaignAs')} withHeader dropdown dataUrl={`rest/campaigns-test-users-table/${this.props.entity.id}`} columns={testUsersColumns} selectionLabelIndex={1} />
+                    <TableSelect id="testUser" label={t('Preview as')} withHeader dropdown dataUrl={`rest/campaigns-test-users-table/${this.props.entity.id}`} columns={testUsersColumns} selectionLabelIndex={1} />
-                <ButtonRow>
-                    <Button className="btn-primary" label={t('preview')} onClickAsync={::this.previewAsync}/>
-                </ButtonRow>
                 </Form>
+            </ModalDialog>
         );
     }
 }
@@ -96,6 +105,12 @@ class TestUser extends Component {
 class SendControls extends Component {
     constructor(props) {
         super(props);
+
+        this.state = {
+            showTestSendModal: false,
+            previewForTestUserVisible: false
+        };
+
         this.initForm({
             leaveConfirmation: false
         });
@@ -257,13 +272,33 @@ class SendControls extends Component {
         const t = this.props.t;
         const entity = this.props.entity;
 
-        const yesNoDialog = (
+        const dialogs = (
+            <>
+                <TestSendModalDialog
+                    mode={TestSendModalDialogMode.CAMPAIGN_STATUS}
+                    visible={this.state.showTestSendModal}
+                    onHide={() => this.setState({showTestSendModal: false})}
+                    campaign={this.props.entity}
+                />
+                <PreviewForTestUserModalDialog
+                    visible={this.state.previewForTestUserVisible}
+                    onHide={() => this.setState({previewForTestUserVisible: false})}
+                    entity={this.props.entity}
+                />
                 <ModalDialog hidden={!this.state.modalVisible} title={this.state.modalTitle} onCloseAsync={() => this.modalAction(false)} buttons={[
                     { label: t('no'), className: 'btn-primary', onClickAsync: () => this.modalAction(false) },
                     { label: t('yes'), className: 'btn-danger', onClickAsync: () => this.modalAction(true) }
                 ]}>
                     {this.state.modalMessage}
                 </ModalDialog>
+            </>
+        );
+
+        const testButtons = (
+            <>
+                <Button className="btn-success" label={t('Preview')} onClickAsync={async () => this.setState({previewForTestUserVisible: true})}/>
+                <Button className="btn-success" label={t('Test send')} onClickAsync={async () => this.setState({showTestSendModal: true})}/>
+            </>
         );
 
         if (entity.status === CampaignStatus.IDLE || entity.status === CampaignStatus.PAUSED || (entity.status === CampaignStatus.SCHEDULED && entity.scheduled)) {
@@ -271,7 +306,7 @@ class SendControls extends Component {
             const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`;
 
             return (
-                <div>{yesNoDialog}
+                <div>{dialogs}
                     <AlignedRow label={t('sendStatus')}>
                         {entity.scheduled ? t('campaignIsScheduledForDelivery') : t('campaignIsReadyToBeSentOut')}
                     </AlignedRow>
@@ -292,20 +327,36 @@ class SendControls extends Component {
                             :
                             <Button className="btn-primary" icon="send" label={t('send') + subscrInfo} onClickAsync={::this.confirmStart}/>
                         }
+                        {entity.status === CampaignStatus.PAUSED && <Button className="btn-primary" icon="refresh" label={t('reset')} onClickAsync={::this.resetAsync}/>}
                         {entity.status === CampaignStatus.PAUSED && <LinkButton className="btn-secondary" icon="signal" label={t('viewStatistics')} to={`/campaigns/${entity.id}/statistics`}/>}
+                        {testButtons}
+                    </ButtonRow>
+                </div>
+            );
+
+        } else if (entity.status === CampaignStatus.PAUSING) {
+            return (
+                <div>{dialogs}
+                    <AlignedRow label={t('sendStatus')}>
+                        {t('Campaign is being paused. Please wait.')}
+                    </AlignedRow>
+                    <ButtonRow>
+                        <LinkButton className="btn-secondary" icon="signal" label={t('viewStatistics')} to={`/campaigns/${entity.id}/statistics`}/>
+                        {testButtons}
                     </ButtonRow>
                 </div>
             );
 
         } else if (entity.status === CampaignStatus.SENDING || (entity.status === CampaignStatus.SCHEDULED && !entity.scheduled)) {
             return (
-                <div>{yesNoDialog}
+                <div>{dialogs}
                     <AlignedRow label={t('sendStatus')}>
                         {t('campaignIsBeingSentOut')}
                     </AlignedRow>
                     <ButtonRow>
                         <Button className="btn-primary" icon="stop" label={t('stop')} onClickAsync={::this.stopAsync}/>
                         <LinkButton className="btn-secondary" icon="signal" label={t('viewStatistics')} to={`/campaigns/${entity.id}/statistics`}/>
+                        {testButtons}
                     </ButtonRow>
                 </div>
             );
@@ -314,7 +365,7 @@ class SendControls extends Component {
             const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`;
 
             return (
-                <div>{yesNoDialog}
+                <div>{dialogs}
                     <AlignedRow label={t('sendStatus')}>
                         {t('allMessagesSent!HitContinueIfYouYouWant')}
                     </AlignedRow>
@@ -322,35 +373,38 @@ class SendControls extends Component {
                         <Button className="btn-primary" icon="play" label={t('continue') + subscrInfo} onClickAsync={::this.confirmStart}/>
                         <Button className="btn-primary" icon="refresh" label={t('reset')} onClickAsync={::this.resetAsync}/>
                         <LinkButton className="btn-secondary" icon="signal" label={t('viewStatistics')} to={`/campaigns/${entity.id}/statistics`}/>
+                        {testButtons}
                     </ButtonRow>
                 </div>
             );
 
         } else if (entity.status === CampaignStatus.INACTIVE) {
             return (
-                <div>{yesNoDialog}
+                <div>{dialogs}
                     <AlignedRow label={t('sendStatus')}>
                         {t('yourCampaignIsCurrentlyDisabledClick')}
                     </AlignedRow>
                     <ButtonRow>
                         <Button className="btn-primary" icon="play" label={t('enable')} onClickAsync={::this.enableAsync}/>
+                        {testButtons}
                     </ButtonRow>
                 </div>
             );
 
         } else if (entity.status === CampaignStatus.ACTIVE) {
             return (
-                <div>{yesNoDialog}
+                <div>{dialogs}
                     <AlignedRow label={t('sendStatus')}>
                         {t('yourCampaignIsEnabledAndSendingMessages')}
                     </AlignedRow>
                     <ButtonRow>
                         <Button className="btn-primary" icon="stop" label={t('disable')} onClickAsync={::this.disableAsync}/>
+                        {testButtons}
                     </ButtonRow>
                 </div>
             );
-        } else {
 
+        } else {
             return null;
         }
     }
@@ -441,7 +495,7 @@ export default class Status extends Component {
             addOverridable('from_name', t('fromName'));
             addOverridable('from_email', t('fromEmailAddress'));
             addOverridable('reply_to', t('replytoEmailAddress'));
-            addOverridable('subject', t('subjectLine'));
+            sendSettings.push(<AlignedRow key="subject" label={t('subjectLine')}>{entity.subject}</AlignedRow>);
         } else {
             sendSettings =  <AlignedRow>{t('loadingSendConfiguration')}</AlignedRow>
         }
@@ -491,13 +545,6 @@ export default class Status extends Component {
                     <Table withHeader dataUrl={`rest/lists-with-segment-by-campaign-table/${this.props.entity.id}`} columns={listsColumns} />
                 </AlignedRow>
 
-                {(entity.type === CampaignType.REGULAR || entity.type === CampaignType.TRIGGERED) &&
-                    <div>
-                        <hr/>
-                        <TestUser entity={entity}/>
-                    </div>
-                }
-
                 <hr/>
                 <SendControls entity={entity} refreshEntity={::this.refreshEntity}/>
 

client/src/campaigns/TestSendModalDialog.js

@@ -5,13 +5,26 @@ import {withTranslation} from '../lib/i18n';
 import PropTypes from 'prop-types';
 import {ModalDialog} from "../lib/bootstrap-components";
 import {requiresAuthenticatedUser, withPageHelpers} from "../lib/page";
-import {Form, TableSelect, withForm} from "../lib/form";
+import {CheckBox, Dropdown, Form, InputField, TableSelect, withForm} from "../lib/form";
 import {withErrorHandling} from "../lib/error-handling";
 import {getMailerTypes} from "../send-configurations/helpers";
 import axios from '../lib/axios';
 import {getUrl} from '../lib/urls';
 import {withComponentMixins} from "../lib/decorator-helpers";
+import {CampaignType} from "../../../shared/campaigns";
 
+const Target = {
+    CAMPAIGN_ONE: 'campaign_one',
+    CAMPAIGN_ALL: 'campaign_all',
+    LIST_ONE: 'list_one',
+    LIST_ALL: 'list_all'
+};
+
+export const TestSendModalDialogMode = {
+    TEMPLATE: 0,
+    CAMPAIGN_CONTENT: 1,
+    CAMPAIGN_STATUS: 2
+}
 
 @withComponentMixins([
     withTranslation,
@@ -27,21 +40,39 @@ export class TestSendModalDialog extends Component {
         this.mailerTypes = getMailerTypes(props.t);
 
         this.initForm({
-            leaveConfirmation: false
+            leaveConfirmation: false,
+            onChangeBeforeValidation: {
+                list: this.onListChanged
+            }
         });
     }
 
     static propTypes = {
-        stateOwner: PropTypes.object,
         visible: PropTypes.bool.isRequired,
+        mode: PropTypes.number.isRequired,
         onHide: PropTypes.func.isRequired,
-        getDataAsync: PropTypes.func.isRequired,
+        getDataAsync: PropTypes.func,
-        entity: PropTypes.object
+        campaign: PropTypes.object
+    }
+
+    onListChanged(mutStateData, key, oldValue, newValue) {
+        mutStateData.setIn(['segment', 'value'], null);
     }
 
     componentDidMount() {
+        const t = this.props.t;
+
         this.populateFormValues({
-            testUser: null,
+            target: Target.CAMPAIGN_ONE,
+            testUserSubscriptionCid: null,
+            testUserListAndSubscriptionCid: null,
+            subjectPrepend: '',
+            subjectAppend: t(' [Test]'),
+            sendConfiguration: null,
+            listCid: null,
+            list: null,
+            segment: null,
+            useSegmentation: false
         });
     }
 
@@ -52,25 +83,49 @@ export class TestSendModalDialog extends Component {
     async performAction() {
         const props = this.props;
         const t = props.t;
+        const mode = this.props.mode;
 
         if (this.isFormWithoutErrors()) {
-
             try {
                 this.hideFormValidation();
                 this.disableForm();
                 this.setFormStatusMessage('info', t('sendingTestEmail'));
 
-                const data = await this.props.getDataAsync();
+                const data = {};
 
-                const campaignCid = props.entity.cid;
+                if (mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.TEMPLATE) {
-                const [listCid, subscriptionCid] = this.getFormValue('testUser').split(':');
+                    const contentData = await this.props.getDataAsync();
+                    data.html = contentData.html;
+                    data.text = contentData.text;
+                }
 
+                if (mode === TestSendModalDialogMode.TEMPLATE) {
+                    data.listCid = this.getFormValue('listCid');
+                    data.subscriptionCid = this.getFormValue('testUserSubscriptionCid');
+                    data.sendConfigurationId = this.getFormValue('sendConfiguration');
+
+                } else if (mode === TestSendModalDialogMode.CAMPAIGN_STATUS || mode === TestSendModalDialogMode.CAMPAIGN_CONTENT) {
+                    data.campaignId = props.campaign.id;
+                    data.subjectPrepend = this.getFormValue('subjectPrepend');
+                    data.subjectAppend = this.getFormValue('subjectAppend');
+
+                    const target = this.getFormValue('target');
+                    if (target === Target.CAMPAIGN_ONE) {
+                        const [listCid, subscriptionCid] = this.getFormValue('testUserListAndSubscriptionCid').split(':');
                         data.listCid = listCid;
                         data.subscriptionCid = subscriptionCid;
-                data.sendConfigurationId = props.entity.send_configuration;
-                data.campaignId = props.entity.id;
 
-                await axios.post(getUrl('rest/template-test-send'), data);
+                    } else if (target === Target.LIST_ALL) {
+                        data.listId = this.getFormValue('list');
+                        data.segmentId = this.getFormValue('useSegmentation') ? this.getFormValue('segment') : null;
+
+                    } else if (target === Target.LIST_ONE) {
+                        data.listCid = this.getFormValue('listCid');
+                        data.subscriptionCid = this.getFormValue('testUserSubscriptionCid');
+                    }
+                }
+
+                await axios.post(getUrl('rest/campaign-test-send'), data);
 
                 this.clearFormStatusMessage();
 
@@ -87,32 +142,204 @@ export class TestSendModalDialog extends Component {
 
     localValidateFormValues(state) {
         const t = this.props.t;
+        const props = this.props;
+        const target = this.getFormValue('target');
+        const mode = this.props.mode;
 
-        if (!state.getIn(['testUser', 'value'])) {
+        state.setIn(['listCid', 'error'], null);
-            state.setIn(['testUser', 'error'], t('subscriptionHasToBeSelected'))
+        state.setIn(['sendConfiguration', 'error'], null);
-        } else {
+        state.setIn(['testUserSubscriptionCid', 'error'], null);
-            state.setIn(['testUser', 'error'], null);
+        state.setIn(['testUserListAndSubscriptionCid', 'error'], null);
+        state.setIn(['list', 'error'], null);
+        state.setIn(['segment', 'error'], null);
+
+        if (mode === TestSendModalDialogMode.TEMPLATE) {
+            if (!state.getIn(['listCid', 'value'])) {
+                state.setIn(['listCid', 'error'], t('listHasToBeSelected'))
+            }
+
+            if (!state.getIn(['sendConfiguration', 'value'])) {
+                state.setIn(['sendConfiguration', 'error'], t('sendConfigurationHasToBeSelected'))
+            }
+
+            if (!state.getIn(['testUserSubscriptionCid', 'value'])) {
+                state.setIn(['testUserSubscriptionCid', 'error'], t('subscriptionHasToBeSelected'))
+            }
+        }
+
+        if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.CAMPAIGN_ONE) {
+            if (!state.getIn(['testUserListAndSubscriptionCid', 'value'])) {
+                state.setIn(['testUserListAndSubscriptionCid', 'error'], t('subscriptionHasToBeSelected'))
+            }
+        }
+
+        if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ONE) {
+            if (!state.getIn(['listCid', 'value'])) {
+                state.setIn(['listCid', 'error'], t('listHasToBeSelected'))
+            }
+
+            if (!state.getIn(['testUserSubscriptionCid', 'value'])) {
+                state.setIn(['testUserSubscriptionCid', 'error'], t('subscriptionHasToBeSelected'))
+            }
+        }
+
+        if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ALL) {
+            if (!state.getIn(['list', 'value'])) {
+                state.setIn(['list', 'error'], t('listMustBeSelected'));
+            }
+
+            if (state.getIn(['useSegmentation', 'value']) && !state.getIn(['segment', 'value'])) {
+                state.setIn(['segment', 'error'], t('segmentMustBeSelected'));
+            }
         }
     }
 
     render() {
         const t = this.props.t;
+        const props = this.props;
+
+        const content = [];
+        const target = this.getFormValue('target');
+        const mode = this.props.mode;
+
+        if (mode === TestSendModalDialogMode.CAMPAIGN_STATUS) {
+            const targetOpts = [
+                {key: Target.CAMPAIGN_ONE, label: t('Single test user of the campaign')},
+                {key: Target.CAMPAIGN_ALL, label: t('All test users of the campaign')},
+                {key: Target.LIST_ONE, label: t('Single test user from a list')},
+                {key: Target.LIST_ALL, label: t('All test users from a list/segment')}
+            ];
+
+            content.push(
+                <Dropdown key="target" id="target" format="wide" label={t('Select to where you want to send the test')} options={targetOpts}/>
+            );
+        }
+
+        if (mode === TestSendModalDialogMode.TEMPLATE) {
+            const listCid = this.getFormValue('listCid');
 
             const testUsersColumns = [
-            { data: 1, title: t('email') },
+                { data: 1, title: t('subscriptionId'), render: data => <code>{data}</code> },
-            { data: 2, title: t('subscriptionId'), render: data => <code>{data}</code> },
+                { data: 2, title: t('email') }
-            { data: 3, title: t('listId'), render: data => <code>{data}</code> },
-            { data: 4, title: t('list') },
-            { data: 5, title: t('listNamespace') }
             ];
 
+            const listsColumns = [
+                { data: 1, title: t('name') },
+                { data: 2, title: t('id'), render: data => <code>{data}</code> },
+                { data: 3, title: t('subscribers') },
+                { data: 4, title: t('description') },
+                { data: 5, title: t('namespace') }
+            ];
+
+            const sendConfigurationsColumns = [
+                { data: 1, title: t('name') },
+                { data: 2, title: t('id'), render: data => <code>{data}</code> },
+                { data: 3, title: t('description') },
+                { data: 4, title: t('type'), render: data => this.mailerTypes[data].typeName },
+                { data: 6, title: t('namespace') }
+            ];
+
+            content.push(
+                <TableSelect key="sendConfiguration" id="sendConfiguration" format="wide" label={t('sendConfiguration')} withHeader dropdown dataUrl='rest/send-configurations-with-send-permission-table' columns={sendConfigurationsColumns} selectionLabelIndex={1} />
+            );
+
+            content.push(
+                <TableSelect key="listCid" id="listCid" format="wide" label={t('list')} withHeader dropdown dataUrl={`rest/lists-table`} columns={listsColumns} selectionKeyIndex={2} selectionLabelIndex={1} />
+            );
+
+            if (listCid) {
+                content.push(
+                    <TableSelect key="testUserSubscriptionCid" id="testUserSubscriptionCid" format="wide" label={t('subscription')} withHeader dropdown dataUrl={`rest/subscriptions-test-user-table/${listCid}`} columns={testUsersColumns} selectionKeyIndex={1} selectionLabelIndex={2} />
+                );
+            }
+        }
+
+        if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.CAMPAIGN_ONE) {
+            const testUsersColumns = [
+                {data: 1, title: t('email')},
+                {data: 2, title: t('subscriptionId'), render: data => <code>{data}</code>},
+                {data: 3, title: t('listId'), render: data => <code>{data}</code>},
+                {data: 4, title: t('list')},
+                {data: 5, title: t('listNamespace')}
+            ];
+
+            content.push(
+                <TableSelect key="testUserListAndSubscriptionCid" id="testUserListAndSubscriptionCid" format="wide" label={t('subscription')} withHeader dropdown dataUrl={`rest/campaigns-test-users-table/${this.props.campaign.id}`} columns={testUsersColumns} selectionLabelIndex={1} />
+            );
+        }
+
+        if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ONE) {
+            const listCid = this.getFormValue('listCid');
+
+            const listsColumns = [
+                { data: 1, title: t('name') },
+                { data: 2, title: t('id'), render: data => <code>{data}</code> },
+                { data: 3, title: t('subscribers') },
+                { data: 4, title: t('description') },
+                { data: 5, title: t('namespace') }
+            ];
+
+            const testUsersColumns = [
+                { data: 1, title: t('subscriptionId'), render: data => <code>{data}</code> },
+                { data: 2, title: t('email') }
+            ];
+
+            content.push(
+                <TableSelect key="listCid" id="listCid" format="wide" label={t('list')} withHeader dropdown dataUrl={`rest/lists-table`} columns={listsColumns} selectionKeyIndex={2} selectionLabelIndex={1} />
+            );
+
+            if (listCid) {
+                content.push(
+                    <TableSelect key="testUserSubscriptionCid" id="testUserSubscriptionCid" format="wide" label={t('subscription')} withHeader dropdown dataUrl={`rest/subscriptions-test-user-table/${listCid}`} columns={testUsersColumns} selectionKeyIndex={1} selectionLabelIndex={2} />
+                );
+            }
+        }
+
+        if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ALL) {
+            const listsColumns = [
+                { data: 1, title: t('name') },
+                { data: 2, title: t('id'), render: data => <code>{data}</code> },
+                { data: 3, title: t('subscribers') },
+                { data: 4, title: t('description') },
+                { data: 5, title: t('namespace') }
+            ];
+
+            const segmentsColumns = [
+                { data: 1, title: t('name') }
+            ];
+
+            content.push(
+                <TableSelect key="list" id="list" format="wide" label={t('list')} withHeader dropdown dataUrl='rest/lists-table' columns={listsColumns} selectionLabelIndex={1} />
+            );
+
+            const selectedList = this.getFormValue('list');
+            content.push(
+                <div key="segment">
+                    <CheckBox id="useSegmentation" format="wide" text={t('useAParticularSegment')}/>
+                    {selectedList && this.getFormValue('useSegmentation') &&
+                        <TableSelect id="segment" format="wide" withHeader dropdown dataUrl={`rest/segments-table/${selectedList}`} columns={segmentsColumns} selectionLabelIndex={1} />
+                    }
+                </div>
+            );
+        }
+
+        if (mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) {
+            content.push(
+                <InputField key="subjectPrepend" id="subjectPrepend" format="wide" label={t('Prepend to subject')}/>
+            );
+
+            content.push(
+                <InputField key="subjectAppend" id="subjectAppend" format="wide" label={t('Append to subject')}/>
+            );
+        }
+
         return (
             <ModalDialog hidden={!this.props.visible} title={t('sendTestEmail')} onCloseAsync={() => this.hideModal()} buttons={[
                 { label: t('send'), className: 'btn-primary', onClickAsync: ::this.performAction },
-                { label: t('cancel'), className: 'btn-danger', onClickAsync: ::this.hideModal }
+                { label: t('close'), className: 'btn-danger', onClickAsync: ::this.hideModal }
             ]}>
                 <Form stateOwner={this} format="wide">
-                    <TableSelect id="testUser" format="wide" label={t('subscription')} withHeader dropdown dataUrl={`rest/campaigns-test-users-table/${this.props.entity.id}`} columns={testUsersColumns} selectionLabelIndex={1} />
+                    {content}
                 </Form>
             </ModalDialog>
         );

client/src/campaigns/helpers.js

@@ -18,7 +18,8 @@ export function getCampaignLabels(t) {
         [CampaignStatus.PAUSED]: t('paused'),
         [CampaignStatus.INACTIVE]: t('inactive'),
         [CampaignStatus.ACTIVE]: t('active'),
-        [CampaignStatus.SENDING]: t('sending')
+        [CampaignStatus.SENDING]: t('sending'),
+        [CampaignStatus.PAUSING]: t('Pausing')
     };
 
 

client/src/lib/form.js

@@ -333,6 +333,13 @@ class InputField extends Component {
 
         const className = owner.addFormValidationClass('form-control', id);
 
+        /* This is for debugging purposes when React reports that InputField is uncontrolled
+        const value = owner.getFormValue(id);
+        if (value === null || value === undefined) console.log(`Warning: InputField ${id} is ${value}`);
+        */
+        const value = owner.getFormValue(id);
+        if (value === null || value === undefined) console.log(`Warning: InputField ${id} is ${value}`);
+
         return wrapInput(id, htmlId, owner, props.format, '', props.label, props.help,
             <input type={type} value={owner.getFormValue(id)} placeholder={props.placeholder} id={htmlId} className={className} aria-describedby={htmlId + '_help'} onChange={evt => owner.updateFormValue(id, evt.target.value)}/>
         );

client/src/send-configurations/CUD.js

@@ -85,7 +85,7 @@ export default class CUD extends Component {
         }
 
         return filterData(data, ['name', 'description', 'from_email', 'from_email_overridable', 'from_name',
-            'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable', 'x_mailer',
+            'from_name_overridable', 'reply_to', 'reply_to_overridable', 'x_mailer',
             'verp_hostname', 'verp_disable_sender_header', 'mailer_type', 'mailer_settings', 'namespace']);
     }
 
@@ -103,8 +103,6 @@ export default class CUD extends Component {
                 from_name_overridable: false,
                 reply_to: '',
                 reply_to_overridable: false,
-                subject: '',
-                subject_overridable: false,
                 verpEnabled: false,
                 verp_hostname: '',
                 verp_disable_sender_header: false,
@@ -233,8 +231,6 @@ export default class CUD extends Component {
                         <CheckBox id="from_name_overridable" text={t('overridable')} className={sendConfigurationsStyles.overridableCheckbox}/>
                         <InputField id="reply_to" label={t('defaultReplytoEmail')}/>
                         <CheckBox id="reply_to_overridable" text={t('overridable')} className={sendConfigurationsStyles.overridableCheckbox}/>
-                        <InputField id="subject" label={t('subject')}/>
-                        <CheckBox id="subject_overridable" text={t('overridable')} className={sendConfigurationsStyles.overridableCheckbox}/>
                         <InputField id="x_mailer" label={t('xMailer')}/>
                     </Fieldset>
 

client/src/send-configurations/helpers.js

@@ -21,7 +21,7 @@ export function getMailerTypes(t) {
         const initVals = mailerTypes[mailerType].initData();
 
         for (const key in initVals) {
-            if (!mutStateData.hasIn([key])) {
+            if (!mutStateData.hasIn([key, 'value']) || mutStateData.getIn([key, 'value']) === undefined) {
                 mutStateData.setIn([key, 'value'], initVals[key]);
             }
         }

client/src/templates/CUD.js

@@ -27,7 +27,7 @@ import {getEditForm, getTemplateTypes, getTypeForm} from './helpers';
 import axios from '../lib/axios';
 import styles from "../lib/styles.scss";
 import {getUrl} from "../lib/urls";
-import {TestSendModalDialog} from "./TestSendModalDialog";
+import {TestSendModalDialog, TestSendModalDialogMode} from "../campaigns/TestSendModalDialog";
 import {withComponentMixins} from "../lib/decorator-helpers";
 import moment from 'moment';
 
@@ -298,6 +298,7 @@ export default class CUD extends Component {
             <div className={this.state.elementInFullscreen ? styles.withElementInFullscreen : ''}>
                 {isEdit &&
                     <TestSendModalDialog
+                        mode={TestSendModalDialogMode.TEMPLATE}
                         visible={this.state.showTestSendModal}
                         onHide={() => this.setState({showTestSendModal: false})}
                         getDataAsync={this.sendModalGetDataHandler}/>

client/src/templates/TestSendModalDialog.js

@@ -1,149 +0,0 @@
-'use strict';
-
-import React, {Component} from 'react';
-import {withTranslation} from '../lib/i18n';
-import PropTypes from 'prop-types';
-import {ModalDialog} from "../lib/bootstrap-components";
-import {requiresAuthenticatedUser, withPageHelpers} from "../lib/page";
-import {Form, TableSelect, withForm} from "../lib/form";
-import {withErrorHandling} from "../lib/error-handling";
-import moment from "moment";
-import {getMailerTypes} from "../send-configurations/helpers";
-import axios from '../lib/axios';
-import {getUrl} from "../lib/urls";
-import {withComponentMixins} from "../lib/decorator-helpers";
-
-
-@withComponentMixins([
-    withTranslation,
-    withForm,
-    withErrorHandling,
-    withPageHelpers,
-    requiresAuthenticatedUser
-])
-export class TestSendModalDialog extends Component {
-    constructor(props) {
-        super(props);
-
-        this.mailerTypes = getMailerTypes(props.t);
-
-        this.initForm({
-            leaveConfirmation: false
-        });
-    }
-
-    static propTypes = {
-        stateOwner: PropTypes.object,
-        visible: PropTypes.bool.isRequired,
-        onHide: PropTypes.func.isRequired,
-        getDataAsync: PropTypes.func.isRequired
-    }
-
-    componentDidMount() {
-        this.populateFormValues({
-            list: null,
-            testUser: null,
-            sendConfiguration: null
-        });
-    }
-
-    async hideModal() {
-        this.props.onHide();
-    }
-
-    async performAction() {
-        const props = this.props;
-        const t = props.t;
-
-        if (this.isFormWithoutErrors()) {
-
-            try {
-                this.hideFormValidation();
-                this.disableForm();
-                this.setFormStatusMessage('info', t('sendingTestEmail'));
-
-                const data = await this.props.getDataAsync();
-                data.listCid = this.getFormValue('list');
-                data.subscriptionCid = this.getFormValue('testUser');
-                data.sendConfigurationId = this.getFormValue('sendConfiguration');
-
-                await axios.post(getUrl('rest/template-test-send'), data);
-
-                this.clearFormStatusMessage();
-
-                this.enableForm();
-                await this.hideModal();
-
-            } catch (err) {
-                throw err;
-            }
-        } else {
-            this.showFormValidation();
-        }
-    }
-
-    localValidateFormValues(state) {
-        const t = this.props.t;
-
-        if (!state.getIn(['sendConfiguration', 'value'])) {
-            state.setIn(['sendConfiguration', 'error'], t('sendConfigurationHasToBeSelected'))
-        } else {
-            state.setIn(['sendConfiguration', 'error'], null);
-        }
-
-        if (!state.getIn(['list', 'value'])) {
-            state.setIn(['list', 'error'], t('listHasToBeSelected'))
-        } else {
-            state.setIn(['list', 'error'], null);
-        }
-
-        if (!state.getIn(['testUser', 'value'])) {
-            state.setIn(['testUser', 'error'], t('subscriptionHasToBeSelected'))
-        } else {
-            state.setIn(['testUser', 'error'], null);
-        }
-    }
-
-    render() {
-        const t = this.props.t;
-
-        const listId = this.getFormValue('list');
-
-        const testUsersColumns = [
-            { data: 1, title: t('subscriptionId'), render: data => <code>{data}</code> },
-            { data: 2, title: t('email') }
-        ];
-
-        const listsColumns = [
-            { data: 1, title: t('name') },
-            { data: 2, title: t('id'), render: data => <code>{data}</code> },
-            { data: 3, title: t('subscribers') },
-            { data: 4, title: t('description') },
-            { data: 5, title: t('namespace') }
-        ];
-
-        const sendConfigurationsColumns = [
-            { data: 1, title: t('name') },
-            { data: 2, title: t('id'), render: data => <code>{data}</code> },
-            { data: 3, title: t('description') },
-            { data: 4, title: t('type'), render: data => this.mailerTypes[data].typeName },
-            { data: 5, title: t('created'), render: data => moment(data).fromNow() },
-            { data: 6, title: t('namespace') }
-        ];
-
-        return (
-            <ModalDialog hidden={!this.props.visible} title={t('sendTestEmail')} onCloseAsync={() => this.hideModal()} buttons={[
-                { label: t('send'), className: 'btn-primary', onClickAsync: ::this.performAction },
-                { label: t('cancel'), className: 'btn-danger', onClickAsync: ::this.hideModal }
-            ]}>
-                <Form stateOwner={this} format="wide">
-                    <TableSelect id="sendConfiguration" format="wide" label={t('sendConfiguration')} withHeader dropdown dataUrl='rest/send-configurations-with-send-permission-table' columns={sendConfigurationsColumns} selectionLabelIndex={1} />
-                    <TableSelect id="list" format="wide" label={t('list')} withHeader dropdown dataUrl={`rest/lists-table`} columns={listsColumns} selectionKeyIndex={2} selectionLabelIndex={1} />
-                    { listId &&
-                        <TableSelect id="testUser" format="wide" label={t('subscription')} withHeader dropdown dataUrl={`rest/subscriptions-test-user-table/${listId}`} columns={testUsersColumns} selectionKeyIndex={1} selectionLabelIndex={2} />
-                    }
-                </Form>
-            </ModalDialog>
-        );
-    }
-}

locales/en-US/common.json

@@ -76,10 +76,10 @@
   "forcesTheRssFeedCheckToImmediatelyCheck": "Forces the RSS feed check to immediately check the campaign with the given CID (in :campaignCid). It works only for RSS campaigns.",
   "sendTransactionalEmail": "Send transactional email",
   "sendSingleEmailByTemplateWithGiven": "Send single email by template with given templateId",
-  "idOfConfigurationUsedToCreateMailer": "ID of configuration used to create mailer instance",
+  "idOfConfigurationUsedToCreateMailer": "ID of configuration used to create mailer instance. If omitted, the default system send configuration is used.",
   "subject": "Subject",
   "dataPassedToTemplateWhenCompilingWith": "Data passed to template when compiling with Handlebars",
-  "mapOfTemplatesubjectVariablesToReplace": "Map of template/subject variables to replace",
+  "mapOfTemplatesubjectVariablesToReplace": "Map of template variables to replace",
   "apiResponseIsAJsonStructureWithErrorAnd": "API response is a JSON structure with <1>error</1> and <3>data</3> properties. If the response <5>error</5> has a value set then the request failed.",
   "youNeedToDefineProperContentTypeWhen": "You need to define proper <1>Content-Type</1> when making a request. You can either use <3>application/x-www-form-urlencoded</3> for normal form data or <5>application/json</5> for a JSON payload. Using <7>multipart/form-data</7> is not supported.",
   "emailMustNotBeEmpty-1": "Email must not be empty",

server/config/default.yaml

@@ -260,9 +260,9 @@ roles:
       permissions: [view, edit, delete, share, createNamespace, createList, createCustomForm, createReport, createReportTemplate, createTemplate, createMosaicoTemplate, createSendConfiguration, createCampaign, manageUsers]
       children:
         sendConfiguration: [viewPublic, viewPrivate, edit, delete, share, sendWithoutOverrides, sendWithAllowedOverrides, sendWithAnyOverrides]
-        list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports]
+        list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports, send, sendToTestUsers]
         customForm: [view, edit, delete, share]
-        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, fetchRss]
+        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, sendToTestUsers, viewStats, fetchRss]
         template: [view, edit, delete, share, viewFiles, manageFiles]
         report: [view, edit, delete, share, execute, viewContent, viewOutput]
         reportTemplate: [view, edit, delete, share, execute]
@@ -275,9 +275,9 @@ roles:
       permissions: [view, edit, delete, share, createNamespace, createList, createCustomForm, createReport, createTemplate, createMosaicoTemplate, createCampaign]
       children:
         sendConfiguration: [viewPublic, sendWithoutOverrides, sendWithAllowedOverrides]
-        list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports]
+        list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports, send, sendToTestUsers]
         customForm: [view, edit, delete, share]
-        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, fetchRss]
+        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, sendToTestUsers, viewStats, fetchRss]
         template: [view, edit, delete, share, viewFiles, manageFiles]
         report: [view, edit, delete, share, execute, viewContent, viewOutput]
         reportTemplate: [view, share, execute]
@@ -286,11 +286,11 @@ roles:
 
     campaignsCreator:
       name: Campaigns Creator
-      description: In the respective namespace, the user has all permissions for templates and campaigns.
+      description: In the respective namespace, the user has all permissions to create and manage templates and campaigns.
       permissions: [view, createTemplate, createCampaign]
       children:
         sendConfiguration: [viewPublic]
-        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, fetchRss]
+        campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, sendToTestUsers, fetchRss]
         template: [view, edit, delete, share, viewFiles, manageFiles]
 
   sendConfiguration:
@@ -307,11 +307,11 @@ roles:
     master:
       name: Master
       description: All permissions
-      permissions: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports]
+      permissions: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports, send, sendToTestUsers]
     campaignsCreator:
       name: Campaigns Creator
-      description: The user can only use the list in setting up a campaign. However, this gives no permission to view subscriptions or to send to the list.
+      description: The user can only use the list in setting up a campaign and to send email to test users. This gives no permission to view subscriptions or to send to the whole list.
-      permissions: [view, viewFields, viewSegments]
+      permissions: [view, viewFields, viewSegments, sendToTestUsers]
 
   customForm:
     master:
@@ -323,11 +323,11 @@ roles:
     master:
       name: Master
       description: All permissions
-      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, manageMessages, fetchRss]
+      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, sendToTestUsers, viewStats, manageMessages, fetchRss]
     campaignsCreator:
       name: Campaigns Creator
       description: The user can setup the campaign but cannot send it.
-      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, fetchRss]
+      permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, sendToTestUsers, fetchRss]
     rssTrigger:
       name: RSS Campaign Trigger
       description: Allows triggering a fetch of an RSS campaign

server/lib/campaign-sender.js

@@ -21,181 +21,128 @@ const htmlToText = require('html-to-text');
 const {getPublicUrl} = require('./urls');
 const blacklist = require('../models/blacklist');
 const libmime = require('libmime');
-const shares = require('../models/shares');
+const { enforce } = require('./helpers');
+
+const MessageType = {
+    REGULAR: 0,
+    TRIGGERED: 1,
+    TEST: 2
+};
 
 class CampaignSender {
     constructor() {
     }
 
-    static async testSend(context, listCid, subscriptionCid, campaignId, sendConfigurationId, html, text) {
+    /*
-        let sendConfiguration, list, fieldsGrouped, campaign, subscriptionGrouped, useVerp, useVerpSenderHeader, mergeTags, attachments;
+        settings is one of:
-
+        - campaignCid / campaignId
-        await knex.transaction(async tx => {
+        or
-            sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), sendConfigurationId, false, true);
+        - sendConfiguration, listId, attachments, html, text, subject
-            list = await lists.getByCidTx(tx, context, listCid);
+     */
-            fieldsGrouped = await fields.listGroupedTx(tx, list.id);
+    async _init(settings) {
-
-            useVerp = config.verp.enabled && sendConfiguration.verp_hostname;
-            useVerpSenderHeader = useVerp && !sendConfiguration.verp_disable_sender_header;
-
-            subscriptionGrouped = await subscriptions.getByCid(context, list.id, subscriptionCid);
-            mergeTags = fields.getMergeTags(fieldsGrouped, subscriptionGrouped);
-
-            if (campaignId) {
-                campaign = await campaigns.getByIdTx(tx, context, campaignId, false, campaigns.Content.WITHOUT_SOURCE_CUSTOM);
-                await campaigns.enforceSendPermissionTx(tx, context, campaign);
-            } else {
-                await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', sendConfigurationId, 'sendWithoutOverrides');
-
-                // This is to fake the campaign for getMessageLinks, which is called inside formatMessage
-                campaign = {
-                    cid: '[CAMPAIGN_ID]'
-                };
-            }
-        });
-
-        const encryptionKeys = [];
-        for (const fld of fieldsGrouped) {
-            if (fld.type === 'gpg' && mergeTags[fld.key]) {
-                encryptionKeys.push(mergeTags[fld.key].trim());
-            }
-        }
-
-        attachments = [];
-        // replace data: images with embedded attachments
-        html = html.replace(/(<img\b[^>]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => {
-            const cid = shortid.generate() + '-attachments';
-            attachments.push({
-                path: dataUri,
-                cid
-            });
-            return prefix + 'cid:' + cid;
-        });
-
-        html = tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, html, true);
-
-        text = (text || '').trim()
-            ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, text)
-            : htmlToText.fromString(html, {wordwrap: 130});
-
-
-        const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
-
-        const getOverridable = key => {
-            return sendConfiguration[key];
-        };
-
-        const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
-
-        const mail = {
-            from: {
-                name: getOverridable('from_name'),
-                address: getOverridable('from_email')
-            },
-            replyTo: getOverridable('reply_to'),
-            xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
-            to: {
-                name: list.to_name === null ? undefined : tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false),
-                address: subscriptionGrouped.email
-            },
-            sender: useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false,
-
-            envelope: useVerp ? {
-                from: campaignAddress + '@' + sendConfiguration.verp_hostname,
-                to: subscriptionGrouped.email
-            } : false,
-
-            headers: {
-                'x-fbl': campaignAddress,
-                // custom header for SparkPost
-                'x-msys-api': JSON.stringify({
-                    campaign_id: campaignAddress
-                }),
-                // custom header for SendGrid
-                'x-smtpapi': JSON.stringify({
-                    unique_args: {
-                        campaign_id: campaignAddress
-                    }
-                }),
-                // custom header for Mailgun
-                'x-mailgun-variables': JSON.stringify({
-                    campaign_id: campaignAddress
-                }),
-                'List-ID': {
-                    prepared: true,
-                    value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>'
-                }
-            },
-            list: {
-                unsubscribe: null
-            },
-            subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false),
-            html,
-            text,
-
-            attachments,
-            encryptionKeys
-        };
-
-
-        let response;
-        try {
-            const info = await mailer.sendMassMail(mail);
-            response = info.response || info.messageId;
-        } catch (err) {
-            response = err.response || err.message;
-        }
-
-        return response;
-    }
-
-
-    async init(settings) {
         this.listsById = new Map(); // listId -> list
         this.listsByCid = new Map(); // listCid -> list
         this.listsFieldsGrouped = new Map(); // listId -> fieldsGrouped
-        this.attachments = [];
 
         await knex.transaction(async tx => {
             if (settings.campaignCid) {
                 this.campaign = await campaigns.rawGetByTx(tx, 'cid', settings.campaignCid);
-            } else {
+
+            } else if (settings.campaignId) {
                 this.campaign = await campaigns.rawGetByTx(tx, 'id', settings.campaignId);
+
+            } else {
+                // We are not within scope of a campaign (i.e. templates in MessageType.TEST message)
+                // This is to fake the campaign for getMessageLinks, which is called inside formatMessage
+                this.campaign = {
+                    cid: '[CAMPAIGN_ID]',
+                    from_name_override: null,
+                    from_email_override: null,
+                    reply_to_override: null
+                };
             }
 
-            const campaign = this.campaign;
+            if (settings.sendConfigurationId) {
+                this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), settings.sendConfigurationId, false, true);
+            } else if (this.campaign.send_configuration) {
+                this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.send_configuration, false, true);
+            } else {
+                enforce(false);
+            }
 
-            this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration, false, true);
+            this.useVerp = config.verp.enabled && this.sendConfiguration.verp_hostname;
+            this.useVerpSenderHeader = this.useVerp && !this.sendConfiguration.verp_disable_sender_header;
 
-            for (const listSpec of campaign.lists) {
+            if (settings.listId) {
+                const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), settings.listId);
+                this.listsById.set(list.id, list);
+                this.listsByCid.set(list.cid, list);
+                this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id));
+
+            } else if (this.campaign.lists) {
+                for (const listSpec of this.campaign.lists) {
                     const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list);
                     this.listsById.set(list.id, list);
                     this.listsByCid.set(list.cid, list);
                     this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id));
                 }
 
-            if (campaign.source === CampaignSource.TEMPLATE) {
+            } else {
-                this.template = await templates.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.data.sourceTemplate, false);
+                enforce(false);
             }
 
-            const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', campaign.id);
+            if (settings.attachments) {
+                this.attachments = settings.attachments;
+
+            } else if (this.campaign.id) {
+                const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', this.campaign.id);
+
+                this.attachments = [];
                 for (const attachment of attachments) {
                     this.attachments.push({
                         filename: attachment.originalname,
-                    path: files.getFilePath('campaign', 'attachment', campaign.id, attachment.filename)
+                        path: files.getFilePath('campaign', 'attachment', this.campaign.id, attachment.filename)
                     });
                 }
 
-            this.useVerp = config.verp.enabled && this.sendConfiguration.verp_hostname;
+            } else {
-            this.useVerpSenderHeader = this.useVerp && !this.sendConfiguration.verp_disable_sender_header;
+                this.attachments = [];
+            }
+
+            if (settings.html !== undefined) {
+                this.html = settings.html;
+                this.text = settings.text;
+            } else if (this.campaign.source === CampaignSource.TEMPLATE) {
+                this.template = await templates.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.data.sourceTemplate, false);
+            }
+
+            if (settings.subject !== undefined) {
+                this.subject = settings.subject;
+            } else if (this.campaign.subject !== undefined) {
+                this.subject = this.campaign.subject;
+            } else {
+                enforce(false);
+            }
         });
     }
 
-    async _getMessage(campaign, list, subscriptionGrouped, mergeTags, replaceDataImgs) {
+    async _getMessage(list, subscriptionGrouped, mergeTags, replaceDataImgs) {
         let html = '';
         let text = '';
         let renderTags = false;
+        const campaign = this.campaign;
 
+        if (this.renderedHtml !== undefined) {
+            html = this.renderedHtml;
+            text = this.renderedText;
+            renderTags = false;
+
+        } else if (this.html !== undefined) {
+            html = this.html;
+            text = this.text;
+            renderTags = true;
+
+        } else {
             if (campaign.source === CampaignSource.URL) {
                 const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
                 for (const key in mergeTags) {
@@ -229,6 +176,7 @@ class CampaignSender {
             }
 
             html = await links.updateLinks(campaign, list, subscriptionGrouped, mergeTags, html);
+        }
 
         const attachments = this.attachments.slice();
         if (replaceDataImgs) {
@@ -273,6 +221,14 @@ class CampaignSender {
         return tags;
     }
 
+    async initByCampaignCid(campaignCid) {
+        await this._init({campaignCid});
+    }
+
+    async initByCampaignId(campaignId) {
+        await this._init({campaignId});
+    }
+
     async getMessage(listCid, subscriptionCid) {
         const list = this.listsByCid.get(listCid);
         const subscriptionGrouped = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid);
@@ -280,20 +236,36 @@ class CampaignSender {
         const campaign = this.campaign;
         const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign));
 
-        return await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, false);
+        return await this._getMessage(list, subscriptionGrouped, mergeTags, false);
     }
 
-    async sendMessageByEmail(listId, email) {
+
-        const subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, email);
+    /*
-        await this._sendMessage(listId, subscriptionGrouped);
+        subData is one of:
+        - queuedMessage
+        or
+        - email, listId
+     */
+    async _sendMessage(subData) {
+        let msgType;
+        let subscriptionGrouped;
+        let listId;
+
+        if (subData.queuedMessage) {
+            const queuedMessage = subData.queuedMessage;
+            msgType = queuedMessage.type;
+            listId = queuedMessage.list;
+            subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, queuedMessage.subscription);
+
+        } else {
+            enforce(subData.email);
+            enforce(subData.listId);
+
+            msgType = MessageType.REGULAR;
+            listId = subData.listId;
+            subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, subData.email);
         }
 
-    async sendMessageBySubscriptionId(listId, subscriptionId) {
-        const subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, subscriptionId);
-        await this._sendMessage(listId, subscriptionGrouped);
-    }
-
-    async _sendMessage(listId, subscriptionGrouped) {
         const email = subscriptionGrouped.email;
 
         if (await blacklist.isBlacklisted(email)) {
@@ -315,7 +287,7 @@ class CampaignSender {
 
         const sendConfiguration = this.sendConfiguration;
 
-        const {html, text, attachments} = await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, true);
+        const {html, text, attachments} = await this._getMessage(list, subscriptionGrouped, mergeTags, true);
 
         const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
 
@@ -380,7 +352,7 @@ class CampaignSender {
             list: {
                 unsubscribe: listUnsubscribe
             },
-            subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false),
+            subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, this.subject, false),
             html,
             text,
 
@@ -435,17 +407,47 @@ class CampaignSender {
             }
 
 
+            if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) {
                 await knex('campaigns').where('id', campaign.id).increment('delivered');
+            }
         } catch (err) {
+            console.log(err);
+
+            /*
+            { Error: connect ECONNREFUSED 127.0.0.1:55871
+    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)
+  cause:
+   { Error: connect ECONNREFUSED 127.0.0.1:55871
+       at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)
+     stack:
+      'Error: connect ECONNREFUSED 127.0.0.1:55871\n    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)',
+     errno: 'ECONNREFUSED',
+     code: 'ECONNECTION',
+     syscall: 'connect',
+     address: '127.0.0.1',
+     port: 55871,
+     command: 'CONN' },
+  isOperational: true,
+  errno: 'ECONNREFUSED',
+  code: 'ECONNECTION',
+  syscall: 'connect',
+  address: '127.0.0.1',
+  port: 55871,
+  command: 'CONN' }
+
+             */
+
             status = SubscriptionStatus.BOUNCED;
             response = err.response || err.message;
+            if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) {
                 await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced');
             }
+        }
 
 
         const now = new Date();
 
-        if (campaign.type === CampaignType.REGULAR || campaign.type === CampaignType.RSS_ENTRY) {
+        if (msgType === MessageType.REGULAR) {
             await knex('campaign_messages').insert({
                 campaign: this.campaign.id,
                 list: list.id,
@@ -457,16 +459,64 @@ class CampaignSender {
                 updated: now
             });
 
-        } else if (campaign.type = CampaignType.TRIGGERED) {
+        } else if (msgType === MessageType.TRIGGERED || msgType === MessageType.TEST) {
+            if (subData.queuedMessage.data.attachments) {
+                for (const attachment of subData.queuedMessage.data.attachments) {
+                    try {
+                        // We ignore any errors here because we already sent the message. Thus we have to mark it as completed to avoid sending it again.
+                        await knex.transaction(async tx => {
+                            await files.unlockTx(tx, 'campaign', 'attachment', attachment.id);
+                        });
+                    } catch (err) {
+                        log.error('CampaignSender', `Error when unlocking attachment ${attachment.id} for ${listId}:${subscriptionGrouped.email} (queuedId: ${subData.queuedMessage.id})`);
+                        log.verbose(err.stack);
+                    }
+                }
+            }
+
             await knex('queued')
-                .where({
+                .where({id: subData.queuedMessage.id})
-                    campaign: this.campaign.id,
-                    list: list.id,
-                    subscription: subscriptionGrouped.id
-                })
                 .del();
         }
     }
+
+    async sendRegularMessage(listId, email) {
+        await this._sendMessage({listId, email});
+    }
 }
 
-module.exports = CampaignSender;
+CampaignSender.sendQueuedMessage = async (queuedMessage) => {
+    const msgData = queuedMessage.data;
+
+    const cs = new CampaignSender();
+    await cs._init({
+        campaignId: msgData.campaignId,
+        listId: queuedMessage.list,
+        sendConfigurationId: queuedMessage.send_configuration,
+        attachments: msgData.attachments,
+        html: msgData.html,
+        text: msgData.text,
+        subject: msgData.subject
+    });
+
+    await cs._sendMessage({queuedMessage});
+};
+
+CampaignSender.queueMessageTx = async (tx, sendConfigurationId, listId, subscriptionId, messageType, messageData) => {
+    if (messageData.attachments) {
+        for (const attachment of messageData.attachments) {
+            await files.lockTx(tx,'campaign', 'attachment', attachment.id);
+        }
+    }
+
+    await tx('queued').insert({
+        send_configuration: sendConfigurationId,
+        list: listId,
+        subscription: subscriptionId,
+        type: messageType,
+        data: JSON.stringify(messageData)
+    });
+};
+
+module.exports.CampaignSender = CampaignSender;
+module.exports.MessageType = MessageType;

server/lib/dependency-helpers.js

@@ -4,7 +4,6 @@ const knex = require('./knex');
 const interoperableErrors = require('../../shared/interoperable-errors');
 const entitySettings = require('./entity-settings');
 const shares = require('../models/shares');
-const { enforce } = require('./helpers');
 
 const defaultNoOfDependenciesReported = 20;
 
@@ -21,7 +20,7 @@ async function ensureNoDependencies(tx, context, id, depSpecs) {
         if (depSpec.query) {
             rows = await depSpec.query(tx).limit(defaultNoOfDependenciesReported + 1);
         } else if (depSpec.column) {
-            rows = await tx(entityType.entitiesTable).where(depSpec.column, id).select(['id', 'name']).limit(defaultNoOfDependenciesReported + 1);
+            rows = await tx(entityType.entitiesTable).where(depSpec.column, id).forShare().select(['id', 'name']).limit(defaultNoOfDependenciesReported + 1);
         } else if (depSpec.rows) {
             rows = await depSpec.rows(tx, defaultNoOfDependenciesReported + 1)
         }

server/lib/entity-settings.js

@@ -44,6 +44,7 @@ const entityTypes = {
             },
             attachment: {
                 table: 'files_campaign_attachment',
+                inUseTable: 'files_campaign_attachment_usage',
                 permissions: {
                     view: 'viewAttachments',
                     manage: 'manageAttachments'

server/lib/helpers.js

@@ -28,11 +28,11 @@ function filterObject(obj, allowedKeys) {
     return result;
 }
 
-function castToInteger(id) {
+function castToInteger(id, msg) {
     const val = parseInt(id);
 
     if (!Number.isInteger(val)) {
-        throw new Error('Invalid id');
+        throw new Error(msg || 'Invalid id');
     }
 
     return val;

server/lib/mailers.js

@@ -94,14 +94,18 @@ async function _sendMail(transport, mail, template) {
     return await trySendAsync();
 }
 
-async function _sendTransactionalMail(transport, mail, template) {
+async function _sendTransactionalMail(transport, mail) {
-    const sendConfiguration = transport.mailer.sendConfiguration;
-
     if (!mail.headers) {
         mail.headers = {};
     }
     mail.headers['X-Sending-Zone'] = 'transactional';
 
+    return await _sendMail(transport, mail);
+}
+
+async function _sendTransactionalMailBasedOnTemplate(transport, mail, template) {
+    const sendConfiguration = transport.mailer.sendConfiguration;
+
     mail.from = {
         name: sendConfiguration.from_name,
         address: sendConfiguration.from_email
@@ -129,7 +133,7 @@ async function _sendTransactionalMail(transport, mail, template) {
         });
     }
 
-    return await _sendMail(transport, mail);
+    return await _sendTransactionalMail(transport, mail);
 }
 
 async function _createTransport(sendConfiguration) {
@@ -263,7 +267,8 @@ async function _createTransport(sendConfiguration) {
     transport.mailer = {
         sendConfiguration,
         throttleWait: bluebird.promisify(throttleWait),
-        sendTransactionalMail: async (mail, template) => await _sendTransactionalMail(transport, mail, template),
+        sendTransactionalMail: async (mail, template) => await _sendTransactionalMail(transport, mail),
+        sendTransactionalMailBasedOnTemplate: async (mail, template) => await _sendTransactionalMailBasedOnTemplate(transport, mail, template),
         sendMassMail: async (mail, template) => await _sendMail(transport, mail)
     };
 

server/lib/subscription-mail-helpers.js

@@ -139,7 +139,7 @@ async function _sendMail(list, email, template, locale, subjectKey, relativeUrls
     try {
         if (list.send_configuration) {
             const mailer = await mailers.getOrCreateMailer(list.send_configuration);
-            await mailer.sendTransactionalMail({
+            await mailer.sendTransactionalMailBasedOnTemplate({
                 to: {
                     name: getDisplayName(flds, subscription),
                     address: email

server/lib/template-sender.js

@@ -1,87 +0,0 @@
-'use strict';
-
-const mailers = require('./mailers');
-const tools = require('./tools');
-const templates = require('../models/templates');
-const { getMergeTagsForBases } = require('../../shared/templates');
-const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
-
-class TemplateSender {
-    constructor(options) {
-        this.defaultOptions = {
-            maxMails: 100,
-            ...options
-        };
-    }
-
-    async send(params) {
-        const options = { ...this.defaultOptions, ...params };
-        this._validateMailOptions(options);
-
-        const [mailer, template] = await Promise.all([
-            mailers.getOrCreateMailer(
-                options.sendConfigurationId
-            ),
-            templates.getById(
-                options.context,
-                options.templateId,
-                false
-            )
-        ]);
-
-        const variables = {
-            EMAIL: options.email,
-            ...getMergeTagsForBases(getTrustedUrl(), getSandboxUrl(), getPublicUrl()),
-            ...options.variables
-        };
-
-        const html = tools.formatTemplate(
-            template.html,
-            null,
-            variables,
-            true
-        );
-        const subject = tools.formatTemplate(
-            options.subject || template.description || template.name,
-            variables
-        );
-        return mailer.sendTransactionalMail(
-            {
-                to: options.email,
-                subject
-            },
-            {
-                html: { template: html },
-                data: options.data,
-                locale: options.locale
-            }
-        );
-    }
-
-    _validateMailOptions(options) {
-        let { context, email, locale, templateId } = options;
-
-        if (!templateId) {
-            throw new Error('Missing templateId');
-        }
-        if (!context) {
-            throw new Error('Missing context');
-        }
-        if (!email || email.length === 0) {
-            throw new Error('Missing email');
-        }
-        if (typeof email === 'string') {
-            email = email.split(',');
-        }
-        if (email.length > options.maxMails) {
-            throw new Error(
-                `Cannot send more than ${options.maxMails} emails at once`
-            );
-        }
-        if (!locale) {
-            throw new Error('Missing locale');
-        }
-    }
-}
-
-module.exports = TemplateSender;

server/models/blacklist.js

@@ -41,16 +41,17 @@ async function search(context, offset, limit, search) {
 async function add(context, email) {
     enforce(email, 'Email has to be set');
 
-    return await knex.transaction(async tx => {
     shares.enforceGlobalPermission(context, 'manageBlacklist');
 
-        const existing = await tx('blacklist').where('email', email).first();
+    try {
-        if (!existing) {
+        await knex('blacklist').insert({email});
-            await tx('blacklist').insert({email});
-        }
-
         await activityLog.logBlacklistActivity(BlacklistActivityType.ADD, email);
-    });
+    } catch (err) {
+        if (err.code === 'ER_DUP_ENTRY') {
+        } else {
+            throw err;
+        }
+    }
 }
 
 async function remove(context, email) {

server/models/campaigns.js

@@ -21,12 +21,14 @@ const {LinkId} = require('./links');
 const feedcheck = require('../lib/feedcheck');
 const contextHelpers = require('../lib/context-helpers');
 const {convertFileURLs} = require('../lib/campaign-content');
+const {CampaignSender, MessageType} = require('../lib/campaign-sender');
+const lists = require('./lists');
 
 const {EntityActivityType, CampaignActivityType} = require('../../shared/activity-log');
 const activityLog = require('../lib/activity-log');
 
 const allowedKeysCommon = ['name', 'description', 'segment', 'namespace',
-    'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url'];
+    'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url'];
 
 const allowedKeysCreate = new Set(['type', 'source', ...allowedKeysCommon]);
 const allowedKeysCreateRssEntry = new Set(['type', 'source', 'parent', ...allowedKeysCommon]);
@@ -168,7 +170,7 @@ async function listTestUsersDTAjax(context, campaignId, params) {
             let subsQry;
 
             if (subsQrys.length === 1) {
-                const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`'
+                const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`';
                 subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
 
             } else {
@@ -342,7 +344,7 @@ async function rawGetByTx(tx, key, id) {
         .groupBy('campaigns.id')
         .select([
             'campaigns.id', 'campaigns.cid', 'campaigns.name', 'campaigns.description', 'campaigns.namespace', 'campaigns.status', 'campaigns.type', 'campaigns.source',
-            'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject_override',
+            'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject',
             'campaigns.data', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled', 'campaigns.unsubscribe_url', 'campaigns.scheduled',
             'campaigns.delivered', 'campaigns.unsubscribed', 'campaigns.bounced', 'campaigns.complained', 'campaigns.blacklisted', 'campaigns.opened', 'campaigns.clicks',
             knex.raw(`GROUP_CONCAT(CONCAT_WS(\':\', campaign_lists.list, campaign_lists.segment) ORDER BY campaign_lists.id SEPARATOR \';\') as lists`)
@@ -632,21 +634,32 @@ async function remove(context, id) {
     });
 }
 
-async function enforceSendPermissionTx(tx, context, campaignId) {
+async function enforceSendPermissionTx(tx, context, campaignOrCampaignId, isToTestUsers, listId) {
     let campaign;
 
-    if (typeof campaignId === 'object') {
+    if (typeof campaignOrCampaignId === 'object') {
-        campaign = campaignId;
+        campaign = campaignOrCampaignId;
     } else {
-        campaign = await getByIdTx(tx, context, campaignId, false);
+        campaign = await getByIdTx(tx, context, campaignOrCampaignId, false);
     }
 
     const sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration, false, false);
 
-    const requiredPermission = getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration);
+    const requiredSendConfigurationPermission = getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration);
+    await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', campaign.send_configuration, requiredSendConfigurationPermission);
 
-    await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', campaign.send_configuration, requiredPermission);
+    const requiredListAndCampaignPermission = isToTestUsers ? 'sendToTestUsers' : 'send';
-    await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaign.id, 'send');
+
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaign.id, requiredListAndCampaignPermission);
+
+    if (listId) {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, requiredListAndCampaignPermission);
+
+    } else {
+        for (const listIds of campaign.lists) {
+            await shares.enforceEntityPermissionTx(tx, context, 'list', listIds.list, requiredListAndCampaignPermission);
+        }
+    }
 }
 
 
@@ -845,12 +858,9 @@ async function getSubscribersQueryGeneratorTx(tx, campaignId) {
 
 async function _changeStatus(context, campaignId, permittedCurrentStates, newState, invalidStateMessage, scheduled = null) {
     await knex.transaction(async tx => {
-        const entity = await tx('campaigns').where('id', campaignId).first();
+        const entity = await getByIdTx(tx, context, campaignId, false);
-        if (!entity) {
-            throw new interoperableErrors.NotFoundError();
-        }
 
-        await enforceSendPermissionTx(tx, context, entity);
+        await enforceSendPermissionTx(tx, context, entity, false);
 
         if (!permittedCurrentStates.includes(entity.status)) {
             throw new interoperableErrors.InvalidStateError(invalidStateMessage);
@@ -869,11 +879,11 @@ async function _changeStatus(context, campaignId, permittedCurrentStates, newSta
 
 
 async function start(context, campaignId, startAt) {
-    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.SCHEDULED, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt);
+    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE, PAUSED, or FINISHED state', startAt);
 }
 
 async function stop(context, campaignId) {
-    await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED], CampaignStatus.PAUSED, 'Cannot stop campaign until it is in SCHEDULED state');
+    await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED, CampaignStatus.SENDING], CampaignStatus.PAUSING, 'Cannot stop campaign until it is in SCHEDULED or SENDING state');
 }
 
 async function reset(context, campaignId) {
@@ -944,6 +954,103 @@ async function fetchRssCampaign(context, cid) {
     });
 }
 
+async function testSend(context, data) {
+    // Though it's a bit counterintuitive, this handles also test sends of a template (i.e. without any campaign id)
+
+    await knex.transaction(async tx => {
+        const processSubscriber = async (sendConfigurationId, listId, subscriptionId, messageData) => {
+            await CampaignSender.queueMessageTx(tx, sendConfigurationId, listId, subscriptionId, MessageType.TEST, messageData);
+
+            await activityLog.logEntityActivity('campaign', CampaignActivityType.TEST_SEND, campaignId, {list: listId, subscription: subscriptionId});
+        };
+
+        const campaignId = data.campaignId;
+
+        if (campaignId) { // This means we are sending a campaign
+            /*
+                Data coming from the client:
+                - html, text
+                - subjectPrepend, subjectAppend
+                - listCid, subscriptionCid
+                - listId, segmentId
+             */
+
+            const campaign = await getByIdTx(tx, context, campaignId, false);
+            const sendConfigurationId = campaign.send_configuration;
+
+            const messageData = {
+                campaignId: campaignId,
+                subject: data.subjectPrepend + campaign.subject + data.subjectAppend,
+                html: data.html, // The html and text may be undefined
+                text: data.text,
+                attachments: []
+            };
+
+            const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', campaignId);
+            for (const attachment of attachments) {
+                messageData.attachments.push({
+                    filename: attachment.originalname,
+                    path: files.getFilePath('campaign', 'attachment', campaign.id, attachment.filename),
+                    id: attachment.id
+                });
+            }
+
+
+            let listId = data.listId;
+            if (!listId && data.listCid) {
+                const list = await lists.getByCidTx(tx, context, data.listCid);
+                listId = list.id;
+            }
+
+            const segmentId = data.segmentId;
+
+            if (listId) {
+                await enforceSendPermissionTx(tx, context, campaign, true, listId);
+
+                if (data.subscriptionCid) {
+                    const subscriber = await subscriptions.getByCidTx(tx, context, listId, data.subscriptionCid);
+                    await processSubscriber(sendConfigurationId, listId, subscriber.id, messageData);
+
+                } else {
+                    const subscribers = await subscriptions.listTestUsersTx(tx, context, listId, segmentId);
+                    for (const subscriber of subscribers) {
+                        await processSubscriber(sendConfigurationId, listId, subscriber.id, messageData);
+                    }
+                }
+
+            } else {
+                for (const lstSeg of campaign.lists) {
+                    await enforceSendPermissionTx(tx, context, campaign, true, lstSeg.list);
+
+                    const subscribers = await subscriptions.listTestUsersTx(tx, context, lstSeg.list, segmentId);
+                    for (const subscriber of subscribers) {
+                        await processSubscriber(sendConfigurationId, lstSeg.list, subscriber.id, messageData);
+                    }
+                }
+            }
+
+        } else { // This means we are sending a template
+            /*
+                Data coming from the client:
+                - html, text
+                - listCid, subscriptionCid, sendConfigurationId
+             */
+
+            const messageData = {
+                subject: 'Test',
+                html: data.html,
+                text: data.text
+            };
+
+            const list = await lists.getByCidTx(tx, context, data.listCid);
+            const subscriber = await subscriptions.getByCidTx(tx, context, list.id, data.subscriptionCid);
+            await processSubscriber(data.sendConfigurationId, list.id, subscriber.id, messageData);
+        }
+    });
+
+    senders.scheduleCheck();
+}
+
 module.exports.Content = Content;
 module.exports.hash = hash;
 
@@ -987,3 +1094,5 @@ module.exports.getTrackingSettingsByCidTx = getTrackingSettingsByCidTx;
 module.exports.getStatisticsOpened = getStatisticsOpened;
 
 module.exports.fetchRssCampaign = fetchRssCampaign;
+
+module.exports.testSend = testSend;

server/models/files.js

@@ -45,7 +45,7 @@ async function listDTAjax(context, type, subType, entityId, params) {
     await shares.enforceEntityPermission(context, type, entityId, getFilesPermission(type, subType, 'view'));
     return await dtHelpers.ajaxList(
         params,
-        builder => builder.from(getFilesTable(type, subType)).where({entity: entityId}),
+        builder => builder.from(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false}),
         ['id', 'originalname', 'filename', 'size', 'created']
     );
 }
@@ -53,7 +53,7 @@ async function listDTAjax(context, type, subType, entityId, params) {
 async function listTx(tx, context, type, subType, entityId) {
     enforceTypePermitted(type, subType);
     await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
-    return await tx(getFilesTable(type, subType)).where({entity: entityId}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc');
+    return await tx(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc');
 }
 
 async function list(context, type, subType, entityId) {
@@ -65,7 +65,7 @@ async function list(context, type, subType, entityId) {
 async function getFileById(context, type, subType, id) {
     enforceTypePermitted(type, subType);
     const file = await knex.transaction(async tx => {
-        const file = await tx(getFilesTable(type, subType)).where('id', id).first();
+        const file = await tx(getFilesTable(type, subType)).where({id: id, delete_pending: false}).first();
         await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'view'));
         return file;
     });
@@ -85,7 +85,7 @@ async function _getFileBy(context, type, subType, entityId, key, value) {
     enforceTypePermitted(type, subType);
     const file = await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
-        const file = await tx(getFilesTable(type, subType)).where({entity: entityId, [key]: value}).first();
+        const file = await tx(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false, [key]: value}).first();
         return file;
     });
 
@@ -155,7 +155,7 @@ async function createFiles(context, type, subType, entityId, files, replacementB
     await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage'));
 
-        const existingNamesRows = await tx(getFilesTable(type, subType)).where('entity', entityId).select(['id', 'filename', 'originalname']);
+        const existingNamesRows = await tx(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false}).select(['id', 'filename', 'originalname']);
 
         const existingNameSet = new Set();
         for (const row of existingNamesRows) {
@@ -275,18 +275,49 @@ async function createFiles(context, type, subType, entityId, files, replacementB
     }
 }
 
-async function removeFile(context, type, subType, id) {
+async function lockTx(tx, type, subType, id) {
+    enforceTypePermitted(type, subType);
+    const filesTableName = getFilesTable(type, subType);
+    await tx(filesTableName).where('id', id).increment('lock_count');
+}
+
+async function unlockTx(tx, type, subType, id) {
     enforceTypePermitted(type, subType);
 
-    const file = await knex.transaction(async tx => {
+    const filesTableName = getFilesTable(type, subType);
-        const file = await tx(getFilesTable(type, subType)).where('id', id).select('entity', 'filename').first();
+    const file = await tx(filesTableName).where('id', id).first();
-        await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'manage'));
+
-        await tx(getFilesTable(type, subType)).where('id', id).del();
+    enforce(file, `File ${id} not found`);
-        return {filename: file.filename, entity: file.entity};
+    enforce(file.lock_count > 0, `Corrupted lock count at file ${id}`);
-    });
+
+    if (file.lock_count === 1 && file.delete_pending) {
+        await tx(filesTableName).where('id', id).del();
 
         const filePath = getFilePath(type, subType, file.entity, file.filename);
         await fs.removeAsync(filePath);
+
+    } else {
+        await tx(filesTableName).where('id', id).update({lock_count: file.lock_count - 1});
+    }
+}
+
+async function removeFile(context, type, subType, id) {
+    enforceTypePermitted(type, subType);
+
+    await knex.transaction(async tx => {
+        const filesTableName = getFilesTable(type, subType);
+        const file = await tx(filesTableName).where('id', id).first();
+        await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'manage'));
+
+        if (!file.lock_count) {
+            await tx(filesTableName).where('id', file.id).del();
+
+            const filePath = getFilePath(type, subType, file.entity, file.filename);
+            await fs.removeAsync(filePath);
+        } else {
+            await tx(filesTableName).where('id', file.id).update({delete_pending: true});
+        }
+    });
 }
 
 async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toType, toSubType, toEntityId) {
@@ -296,7 +327,7 @@ async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toTyp
     enforceTypePermitted(toType, toSubType);
     await shares.enforceEntityPermissionTx(tx, context, toType, toEntityId, getFilesPermission(toType, toSubType, 'manage'));
 
-    const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId});
+    const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId, delete_pending: false});
     for (const row of rows) {
         const fromFilePath = getFilePath(fromType, fromSubType, fromEntityId, row.filename);
         const toFilePath = getFilePath(toType, toSubType, toEntityId, row.filename);
@@ -339,4 +370,6 @@ module.exports.getFileUrl = getFileUrl;
 module.exports.getFilePath = getFilePath;
 module.exports.copyAllTx = copyAllTx;
 module.exports.removeAllTx = removeAllTx;
+module.exports.lockTx = lockTx;
+module.exports.unlockTx = unlockTx;
 module.exports.ReplacementBehavior = ReplacementBehavior;

server/models/links.js

@@ -11,6 +11,7 @@ const he = require('he');
 const { getPublicUrl } = require('../lib/urls');
 const tools = require('../lib/tools');
 const shortid = require('shortid');
+const {enforce} = require('../lib/helpers');
 
 const LinkId = {
     OPEN: -1,
@@ -103,8 +104,7 @@ async function countLink(remoteIp, userAgent, campaignCid, listCid, subscription
 }
 
 async function addOrGet(campaignId, url) {
-    return await knex.transaction(async tx => {
+    const link = await knex('links').select(['id', 'cid']).where({
-        const link = await tx('links').select(['id', 'cid']).where({
         campaign: campaignId,
         url
     }).first();
@@ -112,7 +112,8 @@ async function addOrGet(campaignId, url) {
     if (!link) {
         let cid = shortid.generate();
 
-            const ids = await tx('links').insert({
+        try {
+            const ids = await knex('links').insert({
                 campaign: campaignId,
                 cid,
                 url
@@ -122,10 +123,21 @@ async function addOrGet(campaignId, url) {
                 id: ids[0],
                 cid
             };
+        } catch (err) {
+            if (err.code === 'ER_DUP_ENTRY') {
+                const link = await knex('links').select(['id', 'cid']).where({
+                    campaign: campaignId,
+                    url
+                }).first();
+
+                enforce(link);
+                return link;
+            }
+        }
+
     } else {
         return link;
     }
-    });
 }
 
 async function updateLinks(campaign, list, subscription, mergeTags, message) {

server/models/send-configurations.js

@@ -14,7 +14,7 @@ const mailers = require('../lib/mailers');
 const senders = require('../lib/senders');
 const dependencyHelpers = require('../lib/dependency-helpers');
 
-const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable', 'x_mailer', 'verp_hostname', 'verp_disable_sender_header', 'mailer_type', 'mailer_settings', 'namespace']);
+const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'x_mailer', 'verp_hostname', 'verp_disable_sender_header', 'mailer_type', 'mailer_settings', 'namespace']);
 
 const allowedMailerTypes = new Set(Object.values(MailerType));
 
@@ -75,7 +75,7 @@ async function _getByTx(tx, context, key, id, withPermissions, withPrivateData)
         entity.mailer_settings = JSON.parse(entity.mailer_settings);
     } else {
         entity = await tx('send_configurations').where(key, id).select(
-            ['id', 'name', 'cid', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable']
+            ['id', 'name', 'cid', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable']
         ).first();
 
         if (!entity) {

server/models/subscriptions.js

@@ -19,6 +19,8 @@ const lists = require('./lists');
 
 const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']);
 
+const TEST_USERS_LIST_LIMIT = 1000;
+
 const fieldTypes = {};
 
 const Cardinality = {
@@ -409,6 +411,32 @@ async function list(context, listId, grouped, offset, limit) {
     });
 }
 
+async function listTestUsersTx(tx, context, listId, segmentId, grouped) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+    let entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc').where('is_test', true).limit(TEST_USERS_LIST_LIMIT);
+
+    if (segmentId) {
+        const addSegmentQuery = await segments.getQueryGeneratorTx(tx, listId, segmentId);
+
+        entitiesQry = entitiesQry.where(function() {
+            addSegmentQuery(this);
+        });
+    }
+
+    const entities = await entitiesQry;
+
+    if (grouped) {
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+        for (const entity of entities) {
+            groupSubscription(groupedFieldsMap, entity);
+        }
+    }
+
+    return entities;
+}
+
 // Note that this does not do all the work in the transaction. Thus it is prone to fail if the list is deleted in during the run of the function
 async function* listIterator(context, listId, segmentId, grouped = true) {
     let groupedFieldsMap;
@@ -855,6 +883,7 @@ module.exports.getByEmail = getByEmail;
 module.exports.list = list;
 module.exports.listIterator = listIterator;
 module.exports.listDTAjax = listDTAjax;
+module.exports.listTestUsersTx = listTestUsersTx;
 module.exports.listTestUsersDTAjax = listTestUsersDTAjax;
 module.exports.serverValidate = serverValidate;
 module.exports.create = create;

server/models/templates.js

@@ -7,11 +7,17 @@ const dtHelpers = require('../lib/dt-helpers');
 const interoperableErrors = require('../../shared/interoperable-errors');
 const namespaceHelpers = require('../lib/namespace-helpers');
 const shares = require('./shares');
-const reports = require('./reports');
 const files = require('./files');
 const dependencyHelpers = require('../lib/dependency-helpers');
 const {convertFileURLs} = require('../lib/campaign-content');
 
+const mailers = require('../lib/mailers');
+const tools = require('../lib/tools');
+const sendConfigurations = require('./send-configurations');
+const { getMergeTagsForBases } = require('../../shared/templates');
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const htmlToText = require('html-to-text');
+
 const allowedKeys = new Set(['name', 'description', 'type', 'data', 'html', 'text', 'namespace']);
 
 function hash(entity) {
@@ -145,6 +151,65 @@ async function remove(context, id) {
     });
 }
 
+const MAX_EMAIL_COUNT = 100;
+async function sendAsTransactionalEmail(context, templateId, sendConfigurationId, emails, subject, mergeTags) {
+    // TODO - Update this to use CampaignSender.queueMessageTx (with renderedHtml and renderedText)
+
+    if (emails.length > MAX_EMAIL_COUNT) {
+        throw new Error(`Cannot send more than ${MAX_EMAIL_COUNT} emails at once`);
+    }
+
+    await knex.transaction(async tx => {
+        const template = await getByIdTx(tx, context, templateId,false);
+        const sendConfiguration = await sendConfigurations.getByIdTx(tx, context, sendConfigurationId, false, false);
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', sendConfigurationId, 'sendWithoutOverrides');
+
+        const mailer = await mailers.getOrCreateMailer(sendConfigurationId);
+
+        const variablesSkeleton = {
+            ...getMergeTagsForBases(getTrustedUrl(), getSandboxUrl(), getPublicUrl()),
+            ...mergeTags
+        };
+
+        for (const email of emails) {
+            const variables = {
+                ...variablesSkeleton,
+                EMAIL: email
+            };
+
+            const html = tools.formatTemplate(
+                template.html,
+                null,
+                variables,
+                true
+            );
+
+            const text = (template.text || '').trim()
+                ? tools.formatTemplate(
+                    template.text,
+                    null,
+                    variables,
+                    false
+                ) : htmlToText.fromString(html, {wordwrap: 130});
+
+            return mailer.sendTransactionalMail(
+                {
+                    to: email,
+                    subject,
+                    from: {
+                        name: sendConfiguration.from_name,
+                        address: sendConfiguration.from_email
+                    },
+                    html,
+                    text
+                 }
+            );
+        }
+    });
+}
+
+
 module.exports.hash = hash;
 module.exports.getByIdTx = getByIdTx;
 module.exports.getById = getById;
@@ -153,3 +218,4 @@ module.exports.listByNamespaceDTAjax = listByNamespaceDTAjax;
 module.exports.create = create;
 module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
 module.exports.remove = remove;
+module.exports.sendAsTransactionalEmail = sendAsTransactionalEmail;

server/models/triggers.js

@@ -65,7 +65,7 @@ async function _validateAndPreprocess(tx, context, campaignId, entity) {
         await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.source_campaign, 'view');
     }
 
-    await campaigns.enforceSendPermissionTx(tx, context, campaignId);
+    await campaigns.enforceSendPermissionTx(tx, context, campaignId, false);
 }
 
 async function create(context, campaignId, entity) {

server/models/users.js

@@ -311,7 +311,7 @@ async function sendPasswordReset(locale, usernameOrEmail) {
             const { adminEmail } = await settings.get(contextHelpers.getAdminContext(), ['adminEmail']);
 
             const mailer = await mailers.getOrCreateMailer();
-            await mailer.sendTransactionalMail({
+            await mailer.sendTransactionalMailBasedOnTemplate({
                 to: {
                     address: user.email
                 },

server/routes/api.js

@@ -16,8 +16,10 @@ const contextHelpers = require('../lib/context-helpers');
 const shares = require('../models/shares');
 const slugify = require('slugify');
 const passport = require('../lib/passport');
-const TemplateSender = require('../lib/template-sender');
+const templates = require('../models/templates');
 const campaigns = require('../models/campaigns');
+const {castToInteger} = require('../lib/helpers');
+const {getSystemSendConfigurationId} = require('../../shared/send-configurations');
 
 class APIError extends Error {
     constructor(msg, status) {
@@ -236,7 +238,7 @@ router.postAsync('/blacklist/add', passport.loggedIn, async (req, res) => {
         input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
     });
     if (!(input.EMAIL) || (input.EMAIL === ''))  {
-        throw new Error('EMAIL argument is required');
+        throw new APIError('EMAIL argument is required', 400);
     }
 
     await blacklist.add(req.context, input.EMAIL);
@@ -253,7 +255,7 @@ router.postAsync('/blacklist/delete', passport.loggedIn, async (req, res) => {
         input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
     });
     if (!(input.EMAIL) || (input.EMAIL === '')) {
-        throw new Error('EMAIL argument is required');
+        throw new APIError('EMAIL argument is required', 400);
     }
 
     await blacklist.remove(req.oontext, input.EMAIL);
@@ -288,32 +290,30 @@ router.getAsync('/rss/fetch/:campaignCid', passport.loggedIn, async (req, res) =
 
 router.postAsync('/templates/:templateId/send', async (req, res) => {
     const input = {};
-    Object.keys(req.body).forEach(key => {
-        input[
-            (key || '')
-                .toString()
-                .trim()
-                .toUpperCase()
-        ] = req.body[key] || '';
-    });
 
-    try {
+    for (const key in req.body) {
-        const templateSender = new TemplateSender({
+        const sanitizedKey = key.toString().trim().toUpperCase();
-            context: req.context,
+        input[sanitizedKey] = req.body[key] || '';
-            locale: req.locale,
-            templateId: req.params.templateId
-        });
-        const info = await templateSender.send({
-            data: input.DATA,
-            email: input.EMAIL,
-            sendConfigurationId: input.SEND_CONFIGURATION_ID,
-            subject: input.SUBJECT,
-            variables: input.VARIABLES
-        });
-        res.status(200).json({ data: info });
-    } catch (e) {
-        throw new APIError(e.message, 400);
     }
+
+    const templateId = castToInteger(req.params.templateId, 'Invalid template ID');
+
+    let sendConfigurationId;
+    if (!('SEND_CONFIGURATION_ID' in input)) {
+        sendConfigurationId = getSystemSendConfigurationId();
+    } else {
+        sendConfigurationId = castToInteger(input.SEND_CONFIGURATION_ID, 'Invalid send configuration ID');
+    }
+
+    if (!input.EMAIL || input.EMAIL === 0) {
+        throw new APIError('Missing email(s)', 400);
+    }
+
+    const emails = input.EMAIL.split(',');
+
+    const info = await templates.sendAsTransactionalEmail(req.context, templateId, sendConfigurationId, emails, input.SUBJECT, input.VARIABLES);
+
+    res.json({ data: info });
 });
 
 module.exports = router;

server/routes/archive.js

@@ -1,12 +1,12 @@
 'use strict';
 
 const router = require('../lib/router-async').create();
-const CampaignSender = require('../lib/campaign-sender');
+const {CampaignSender} = require('../lib/campaign-sender');
 
 
 router.get('/:campaign/:list/:subscription', (req, res, next) => {
     const cs = new CampaignSender();
-    cs.init({campaignCid: req.params.campaign})
+    cs.initByCampaignCid(req.params.campaign)
         .then(() => cs.getMessage(req.params.list, req.params.subscription))
         .then(result => {
             const {html} = result;

server/routes/rest/campaigns.js

@@ -114,6 +114,12 @@ router.postAsync('/campaigns-link-clicks-table/:campaignId', passport.loggedIn,
     return res.json(await campaigns.listLinkClicksDTAjax(req.context, castToInteger(req.params.campaignId), req.body));
 });
 
+router.postAsync('/campaign-test-send', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const data = req.body;
+    const result = await campaigns.testSend(req.context, data);
+    return res.json(result);
+});
+
 
 
 module.exports = router;

server/routes/rest/templates.js

@@ -5,7 +5,6 @@ const templates = require('../../models/templates');
 
 const router = require('../../lib/router-async').create();
 const {castToInteger} = require('../../lib/helpers');
-const CampaignSender = require('../../lib/campaign-sender');
 
 
 router.getAsync('/templates/:templateId', passport.loggedIn, async (req, res) => {
@@ -39,10 +38,4 @@ router.postAsync('/templates-by-namespace-table/:namespaceId', passport.loggedIn
     return res.json(await templates.listByNamespaceDTAjax(req.context, castToInteger(req.params.namespaceId), req.body));
 });
 
-router.postAsync('/template-test-send', passport.loggedIn, passport.csrfProtection, async (req, res) => {
-    const data = req.body;
-    const result = await CampaignSender.testSend(req.context, data.listCid, data.subscriptionCid, data.campaignId, data.sendConfigurationId, data.html, data.text);
-    return res.json(result);
-});
-
 module.exports = router;

server/services/feedcheck.js

@@ -112,7 +112,7 @@ async function run() {
                             from_name_override: rssCampaign.from_name_override,
                             from_email_override: rssCampaign.from_email_override,
                             reply_to_override: rssCampaign.reply_to_override,
-                            subject_override: rssCampaign.subject_override,
+                            subject: rssCampaign.subject,
                             data: campaignData,
 
                             click_tracking_disabled: rssCampaign.click_tracking_disabled,

server/services/sender-master.js

@@ -6,35 +6,63 @@ const log = require('../lib/log');
 const path = require('path');
 const knex = require('../lib/knex');
 const {CampaignStatus, CampaignType} = require('../../shared/campaigns');
-const { enforce } = require('../lib/helpers');
 const campaigns = require('../models/campaigns');
 const builtinZoneMta = require('../lib/builtin-zone-mta');
 const {CampaignActivityType} = require('../../shared/activity-log');
 const activityLog = require('../lib/activity-log');
+const {MessageType} = require('../lib/campaign-sender')
 require('../lib/fork');
 
+class Notifications {
+    constructor() {
+        this.conts = new Map();
+    }
+
+    notify(id) {
+        const cont = this.conts.get(id);
+        if (cont) {
+            for (const cb of cont) {
+                setImmediate(cb);
+            }
+            this.conts.delete(id);
+        }
+    }
+
+    async waitFor(id) {
+        let cont = this.conts.get(id);
+        if (!cont) {
+            cont = [];
+        }
+
+        const notified = new Promise(resolve => {
+            cont.push(resolve);
+        });
+
+        this.conts.set(id, cont);
+
+        await notified;
+    }
+}
+
+const notifier = new Notifications();
 
 let messageTid = 0;
 const workerProcesses = new Map();
 
+const workersCount = config.queue.processes;
 const idleWorkers = [];
 
 let campaignSchedulerRunning = false;
 let queuedSchedulerRunning = false;
-let workerSchedulerRunning = false;
 
-const campaignsCheckPeriod = 5 * 1000;
+const campaignsCheckPeriod = 30 * 1000;
 const retrieveBatchSize = 1000;
-const workerBatchSize = 100;
+const workerBatchSize = 10;
 
-const messageQueue = new Map(); // campaignId -> [{listId, email}]
+const sendConfigurationMessageQueue = new Map(); // sendConfigurationId -> [{queuedMessage}]
-const messageQueueCont = new Map(); // campaignId -> next batch callback
+const campaignMessageQueue = new Map(); // campaignId -> [{listId, email}]
-const campaignFinishCont = new Map(); // campaignId -> worker finished callback
 
-const workAssignment = new Map(); // workerId -> { campaignId, subscribers: [{listId, email}] }
+const workAssignment = new Map(); // workerId -> { campaignId, messages: [{listId, email} } / { sendConfigurationId, messages: [{queuedMessage}] }
-
-let workerSchedulerCont = null;
-let queuedLastId = 0;
 
 
 function messagesProcessed(workerId) {
@@ -43,108 +71,151 @@ function messagesProcessed(workerId) {
     workAssignment.delete(workerId);
     idleWorkers.push(workerId);
 
-    if (workerSchedulerCont) {
+    notifier.notify('workerFinished');
-        const cont = workerSchedulerCont;
-        setImmediate(workerSchedulerCont);
-        workerSchedulerCont = null;
-    }
-
-    if (campaignFinishCont.has(wa.campaignId)) {
-        setImmediate(campaignFinishCont.get(wa.campaignId));
-        campaignFinishCont.delete(wa.campaignId);
-    }
 }
 
-async function scheduleWorkers() {
+async function workersLoop() {
+    const reservedWorkersForTestCount = workersCount > 1 ? 1 : 0;
+
     async function getAvailableWorker() {
-        if (idleWorkers.length > 0) {
+        while (idleWorkers.length === 0) {
+            await notifier.waitFor('workerFinished');
+        }
+
         return idleWorkers.shift();
+    }
+
+    function assignCampaignTaskToWorker(workerId, task) {
+        const campaignId = task.campaignId;
+        const queue = task.queue;
+
+        const messages = queue.splice(0, workerBatchSize);
+        workAssignment.set(workerId, {campaignId, messages});
+
+        if (queue.length === 0) {
+            notifier.notify(`campaignMessageQueueEmpty:${campaignId}`);
+        }
+
+        sendToWorker(workerId, 'process-campaign-messages', {
+            campaignId,
+            messages
+        });
+    }
+
+    function assignSendConfigurationTaskToWorker(workerId, task) {
+        const sendConfigurationId = task.sendConfigurationId;
+        const queue = task.queue;
+
+        const messages = queue.splice(0, workerBatchSize);
+        workAssignment.set(workerId, {sendConfigurationId, messages});
+
+        if (queue.length === 0) {
+            notifier.notify(`sendConfigurationMessageQueueEmpty:${sendConfigurationId}`);
+        }
+
+        sendToWorker(workerId, 'process-queued-messages', {
+            sendConfigurationId,
+            messages
+        });
+    }
+
+    function selectNextTask() {
+        const allocationMap = new Map();
+        const allocation = [];
+
+        function initAllocation(attrName, queues, assignWorkerHandler) {
+            for (const id of queues.keys()) {
+                const key = attrName + ':' + id;
+
+                const queue = queues.get(id);
+
+                const task = {
+                    [attrName]: id,
+                    existingWorkers: 0,
+                    isEmpty: queue.length === 0,
+                    queue,
+                    assignWorkerHandler
+                };
+
+                allocationMap.set(key, task);
+                allocation.push(task);
+            }
+
+            for (const wa of workAssignment.values()) {
+                if (wa[attrName]) {
+                    const key = attrName + ':' + wa[attrName];
+                    const task = allocationMap.get(key);
+                    task.existingWorkers += 1;
+                }
+            }
+        }
+
+        initAllocation('sendConfigurationId', sendConfigurationMessageQueue, assignSendConfigurationTaskToWorker);
+        initAllocation('campaignId', campaignMessageQueue, assignCampaignTaskToWorker);
+
+        let minTask = null;
+        let minExistingWorkers;
+
+        for (const task of allocation) {
+            if (!task.isEmpty && (minTask === null || minExistingWorkers > task.existingWorkers)) {
+                minTask = task;
+                minExistingWorkers = task.existingWorkers;
+            }
+        }
+
+        return minTask;
+    }
+
+
+    while (true) {
+        const task = selectNextTask();
+
+        if (task) {
+            const workerId = await getAvailableWorker();
+            task.assignWorkerHandler(workerId, task);
             
         } else {
-            const workerAvailable = new Promise(resolve => {
+            await notifier.waitFor('workAvailable');
-                workerSchedulerCont = resolve;
-            });
-
-            await workerAvailable;
-            return idleWorkers.shift();
         }
     }
-
-
-    if (workerSchedulerRunning) {
-        return;
-    }
-
-    workerSchedulerRunning = true;
-    let workerId = await getAvailableWorker();
-
-    let keepLooping = true;
-
-    while (keepLooping) {
-        keepLooping = false;
-
-        for (const campaignId of messageQueue.keys()) {
-            const queue = messageQueue.get(campaignId);
-
-            if (queue.length > 0) {
-                const subscribers = queue.splice(0, workerBatchSize);
-                workAssignment.set(workerId, {campaignId, subscribers});
-
-                if (queue.length === 0 && messageQueueCont.has(campaignId)) {
-                    setImmediate(messageQueueCont.get(campaignId));
-                    messageQueueCont.delete(campaignId);
-                }
-
-                sendToWorker(workerId, 'process-messages', {
-                    campaignId,
-                    subscribers
-                });
-                workerId = await getAvailableWorker();
-
-                keepLooping = true;
-            }
-        }
-    }
-
-    idleWorkers.push(workerId);
-
-    workerSchedulerRunning = false;
 }
 
 
 async function processCampaign(campaignId) {
-    async function finish() {
+    const msgQueue = campaignMessageQueue.get(campaignId);
+
+    async function finish(newStatus) {
+        const isCompleted = () => {
+            if (msgQueue.length > 0) return false;
+
             let workerRunning = false;
+
             for (const wa of workAssignment.values()) {
                 if (wa.campaignId === campaignId) {
                     workerRunning = true;
                 }
             }
 
-        if (workerRunning) {
+            return !workerRunning;
-            const workerFinished = new Promise(resolve => {
+        };
-                campaignFinishCont.set(campaignId, resolve);
-            });
 
-            await workerFinished;
+        while (!isCompleted()) {
-            setImmediate(finish);
+            await notifier.waitFor('workerFinished');
         }
 
-        await knex('campaigns').where('id', campaignId).update({status: CampaignStatus.FINISHED});
+        campaignMessageQueue.delete(campaignId);
-        await activityLog.logEntityActivity('campaign', CampaignActivityType.STATUS_CHANGE, campaignId, {status: CampaignStatus.FINISHED});
 
-        messageQueue.delete(campaignId);
+        await knex('campaigns').where('id', campaignId).update({status: newStatus});
+        await activityLog.logEntityActivity('campaign', CampaignActivityType.STATUS_CHANGE, campaignId, {status: newStatus});
     }
 
-    const msgQueue = [];
-    messageQueue.set(campaignId, msgQueue);
-
     try {
         while (true) {
             const cpg = await knex('campaigns').where('id', campaignId).first();
 
-            if (cpg.status === CampaignStatus.PAUSED) {
+            if (cpg.status === CampaignStatus.PAUSING) {
-                messageQueue.delete(campaignId);
+                msgQueue.splice(0);
+                await finish(CampaignStatus.PAUSED);
                 return;
             }
 
@@ -154,21 +225,21 @@ async function processCampaign(campaignId) {
             });
 
             if (qryGen) {
-                let subscribersInProcessing = [...msgQueue];
+                let messagesInProcessing = [...msgQueue];
                 for (const wa of workAssignment.values()) {
                     if (wa.campaignId === campaignId) {
-                        subscribersInProcessing = subscribersInProcessing.concat(wa.subscribers);
+                        messagesInProcessing = messagesInProcessing.concat(wa.messages);
                     }
                 }
 
                 const qry = qryGen(knex)
-                    .whereNotIn('pending_subscriptions.email', subscribersInProcessing.map(x => x.email))
+                    .whereNotIn('pending_subscriptions.email', messagesInProcessing.map(x => x.email))
                     .select(['pending_subscriptions.email', 'campaign_lists.list'])
                     .limit(retrieveBatchSize);
                 const subs = await qry;
 
                 if (subs.length === 0) {
-                    await finish();
+                    await finish(CampaignStatus.FINISHED);
                     return;
                 }
 
@@ -179,21 +250,19 @@ async function processCampaign(campaignId) {
                     });
                 }
 
-                const nextBatchNeeded = new Promise(resolve => {
+                notifier.notify('workAvailable');
-                    messageQueueCont.set(campaignId, resolve);
-                });
 
-                setImmediate(scheduleWorkers);
+                while (msgQueue.length > 0) {
-
+                    await notifier.waitFor(`campaignMessageQueueEmpty:${campaignId}`);
-                await nextBatchNeeded;
+                }
 
             } else {
-                await finish();
+                await finish(CampaignStatus.FINISHED);
                 return;
             }
         }
     } catch (err) {
-        log.error('Senders', `Sending campaign ${campaignId} failed with error: ${err.message}`)
+        log.error('Senders', `Sending campaign ${campaignId} failed with error: ${err.message}`);
         log.verbose(err.stack);
     }
 }
@@ -226,6 +295,8 @@ async function scheduleCampaigns() {
             });
 
             if (campaignId) {
+                campaignMessageQueue.set(campaignId, []);
+
                 // noinspection JSIgnoredPromiseFromCall
                 processCampaign(campaignId);
 
@@ -234,16 +305,81 @@ async function scheduleCampaigns() {
             }
         }
     } catch (err) {
-        log.error('Senders', `Scheduling campaigns failed with error: ${err.message}`)
+        log.error('Senders', `Scheduling campaigns failed with error: ${err.message}`);
         log.verbose(err.stack);
     }
 
-
     campaignSchedulerRunning = false;
 }
 
 
-async function processQueued() {
+async function processQueuedBySendConfiguration(sendConfigurationId) {
+    const msgQueue = sendConfigurationMessageQueue.get(sendConfigurationId);
+
+    const isCompleted = () => {
+        if (msgQueue.length > 0) return false;
+
+        let workerRunning = false;
+
+        for (const wa of workAssignment.values()) {
+            if (wa.sendConfigurationId === sendConfigurationId) {
+                workerRunning = true;
+            }
+        }
+
+        return !workerRunning;
+    };
+
+    try {
+        while (true) {
+            let messagesInProcessing = [...msgQueue];
+            for (const wa of workAssignment.values()) {
+                if (wa.sendConfigurationId === sendConfigurationId) {
+                    messagesInProcessing = messagesInProcessing.concat(wa.messages);
+                }
+            }
+
+            const rows = await knex('queued')
+                .orderByRaw(`FIELD(type, ${MessageType.TRIGGERED}, ${MessageType.TEST}) DESC, id ASC`) // This orders MessageType.TEST messages before MessageType.TRIGGERED ones
+                .where('send_configuration', sendConfigurationId)
+                .whereNotIn('id', messagesInProcessing.map(x => x.queuedMessage.id))
+                .limit(retrieveBatchSize);
+
+            if (rows.length === 0) {
+                if (isCompleted()) {
+                    sendConfigurationMessageQueue.delete(sendConfigurationId);
+                    return;
+
+                } else {
+                    while (!isCompleted()) {
+                        await notifier.waitFor('workerFinished');
+                    }
+
+                    // At this point, there might be new messages in the queued that could belong to us. Thus we have to try again instead for returning.
+                    continue;
+                }
+            }
+
+            for (const row of rows) {
+                row.data = JSON.parse(row.data);
+                msgQueue.push({
+                    queuedMessage: row
+                });
+            }
+
+            notifier.notify('workAvailable');
+
+            while (msgQueue.length > 0) {
+                await notifier.waitFor(`sendConfigurationMessageQueueEmpty:${sendConfigurationId}`);
+            }
+        }
+    } catch (err) {
+        log.error('Senders', `Sending queued messages for send configuration ${sendConfigurationId} failed with error: ${err.message}`);
+        log.verbose(err.stack);
+    }
+}
+
+async function scheduleQueued() {
     if (queuedSchedulerRunning) {
         return;
     }
@@ -252,34 +388,23 @@ async function processQueued() {
 
     try {
         while (true) {
-            const rows = await knex('queued')
+            const sendConfigurationsInProcessing = [...sendConfigurationMessageQueue.keys()];
-                .orderBy('id', 'asc')
-                .where('id', '>', queuedLastId)
-                .limit(retrieveBatchSize);
 
-            if (rows.length === 0) {
+            const rows = await knex('queued')
-                break;
+                .whereNotIn('send_configuration', sendConfigurationsInProcessing)
-            }
+                .groupBy('send_configuration')
+                .select(['send_configuration']);
 
             for (const row of rows) {
-                let msgQueue = messageQueue.get(row.campaign);
+                const sendConfigurationId = row.send_configuration;
-                if (!msgQueue) {
+                sendConfigurationMessageQueue.set(sendConfigurationId, []);
-                    msgQueue = [];
+
-                    messageQueue.set(row.campaign, msgQueue);
+                // noinspection JSIgnoredPromiseFromCall
+                processQueuedBySendConfiguration(sendConfigurationId);
             }
-
-                msgQueue.push({
-                    listId: row.list,
-                    subscriptionId: row.subscription
-                });
-            }
-
-            queuedLastId = rows[rows.length - 1].id;
-
-            setImmediate(scheduleWorkers);
         }
     } catch (err) {
-        log.error('Senders', `Processing queued messages failed with error: ${err.message}`)
+        log.error('Senders', `Scheduling queued messages failed with error: ${err.message}`);
         log.verbose(err.stack);
     }
 
@@ -337,7 +462,7 @@ function periodicCampaignsCheck() {
     scheduleCampaigns();
 
     // noinspection JSIgnoredPromiseFromCall
-    processQueued();
+    scheduleQueued();
 
     setTimeout(periodicCampaignsCheck, campaignsCheckPeriod);
 }
@@ -345,7 +470,7 @@ function periodicCampaignsCheck() {
 async function init() {
     const spawnWorkerFutures = [];
     let workerId;
-    for (workerId = 0; workerId < config.queue.processes; workerId++) {
+    for (workerId = 0; workerId < workersCount; workerId++) {
         spawnWorkerFutures.push(spawnWorker(workerId));
     }
 
@@ -358,6 +483,7 @@ async function init() {
             if (type === 'schedule-check') {
                 // noinspection JSIgnoredPromiseFromCall
                 scheduleCampaigns();
+                scheduleQueued();
 
             } else if (type === 'reload-config') {
                 for (const workerId of workerProcesses.keys()) {
@@ -376,6 +502,8 @@ async function init() {
     });
 
     periodicCampaignsCheck();
+
+    setImmediate(workersLoop);
 }
 
 // noinspection JSIgnoredPromiseFromCall

server/services/sender-worker.js

@@ -3,14 +3,13 @@
 const config = require('config');
 const log = require('../lib/log');
 const mailers = require('../lib/mailers');
-const CampaignSender = require('../lib/campaign-sender');
+const {CampaignSender} = require('../lib/campaign-sender');
-const {enforce} = require('../lib/helpers');
 require('../lib/fork');
 
 const workerId = Number.parseInt(process.argv[2]);
 let running = false;
 
-async function processMessages(campaignId, subscribers) {
+async function processCampaignMessages(campaignId, messages) {
     if (running) {
         log.error('Senders', `Worker ${workerId} assigned work while working`);
         return;
@@ -19,23 +18,40 @@ async function processMessages(campaignId, subscribers) {
     running = true;
 
     const cs = new CampaignSender();
-    await cs.init({campaignId})
+    await cs.initByCampaignId(campaignId);
 
-    for (const subData of subscribers) {
+    for (const msgData of messages) {
         try {
-            if (subData.email) {
+            await cs.sendRegularMessage(msgData.listId, msgData.email);
-                await cs.sendMessageByEmail(subData.listId, subData.email);
 
-            } else if (subData.subscriptionId) {
+            log.verbose('Senders', 'Message sent and status updated for %s:%s', msgData.listId, msgData.email);
-                await cs.sendMessageBySubscriptionId(subData.listId, subData.subscriptionId);
+        } catch (err) {
-
+            log.error('Senders', `Sending message to ${msgData.listId}:${msgData.email} failed with error: ${err.message}`);
-            } else {
+            log.verbose(err.stack);
-                enforce(false);
+        }
     }
 
-            log.verbose('Senders', 'Message sent and status updated for %s:%s', subData.listId, subData.email || subData.subscriptionId);
+    running = false;
+
+    sendToMaster('messages-processed');
+}
+
+async function processQueuedMessages(sendConfigurationId, messages) {
+    if (running) {
+        log.error('Senders', `Worker ${workerId} assigned work while working`);
+        return;
+    }
+
+    running = true;
+
+    for (const msgData of messages) {
+        const queuedMessage = msgData.queuedMessage;
+        try {
+            await CampaignSender.sendQueuedMessage(queuedMessage);
+
+            log.verbose('Senders', 'Message sent and status updated for %s:%s', queuedMessage.list, queuedMessage.subscription);
         } catch (err) {
-            log.error('Senders', `Sending message to ${subData.listId}:${subData.email} failed with error: ${err.message}`)
+            log.error('Senders', `Sending message to ${queuedMessage.list}:${queuedMessage.subscription} failed with error: ${err.message}`);
             log.verbose(err.stack);
         }
     }
@@ -58,11 +74,14 @@ process.on('message', msg => {
         if (type === 'reload-config') {
             mailers.invalidateMailer(msg.data.sendConfigurationId);
 
-        } else if (type === 'process-messages') {
+        } else if (type === 'process-campaign-messages') {
             // noinspection JSIgnoredPromiseFromCall
-            processMessages(msg.data.campaignId, msg.data.subscribers)
+            processCampaignMessages(msg.data.campaignId, msg.data.messages)
-        }
 
+        } else if (type === 'process-queued-messages') {
+            // noinspection JSIgnoredPromiseFromCall
+            processQueuedMessages(msg.data.sendConfigurationId, msg.data.messages)
+        }
     }
 });
 

server/services/triggers.js

@@ -10,6 +10,7 @@ const { Entity, Event } = require('../../shared/triggers');
 const { SubscriptionStatus } = require('../../shared/lists');
 const links = require('../models/links');
 const contextHelpers = require('../lib/context-helpers');
+const {MessageType, CampaignSender} = require('../lib/campaign-sender');
 
 const triggerCheckPeriod = 30 * 1000;
 const triggerFirePeriod = 120 * 1000;
@@ -151,12 +152,13 @@ async function run() {
                         subscription: subscriber.id
                     });
 
-                    await tx('queued').insert({
+                    await CampaignSender.queueMessageTx(tx,
-                        campaign: campaign.id,
+                        campaign.send_configuration, cpgList.list, subscriber.id, MessageType.TRIGGERED,
-                        list: cpgList.list,
+                        {
-                        subscription: subscriber.id,
+                            campaignId: campaign.id,
-                        trigger: trigger.id
+                            triggerId: trigger.id
-                    });
+                        }
+                    );
 
                     await tx('triggers').increment('count').where('id', trigger.id);
 

server/setup/knex/migrations/20170506102634_v1_to_v2.js

@@ -753,11 +753,11 @@ async function migrateSettings(knex) {
 
         if (settings.dkimApiKey) {
             mailer_type = MailerType.ZONE_MTA;
-            mailer_settings.dkimApiKey = settings.dkimApiKey;
+            mailer_settings.dkimApiKey = settings.dkimApiKey || '';
             mailer_settings.zoneMtaType = ZoneMTAType.WITH_HTTP_CONF;
-            mailer_settings.dkimDomain = settings.dkimDomain;
+            mailer_settings.dkimDomain = settings.dkimDomain || '';
-            mailer_settings.dkimSelector = settings.dkimSelector;
+            mailer_settings.dkimSelector = settings.dkimSelector || '';
-            mailer_settings.dkimPrivateKey = settings.dkimPrivateKey;
+            mailer_settings.dkimPrivateKey = settings.dkimPrivateKey || '';
         }
     }
 
@@ -777,7 +777,7 @@ async function migrateSettings(knex) {
         verp_hostname: settings.verpUse ? settings.verpHostname : null,
         mailer_type,
         mailer_settings: JSON.stringify(mailer_settings),
-        x_mailer: settings.x_mailer,
+        x_mailer: settings.x_mailer || '',
         namespace: getGlobalNamespaceId()
     });
 
@@ -810,7 +810,7 @@ async function addFiles(knex) {
                 table.string('mimetype');
                 table.integer('size');
                 table.timestamp('created').defaultTo(knex.fn.now());
-                table.index(['entity', 'originalname'])
+                table.index(['entity', 'originalname']);
             });
         }
     }

server/setup/knex/migrations/20190615000000_generalization_of_queued_and_file_locking.js

@@ -0,0 +1,63 @@
+const entityTypesWithFiles = {
+    campaign: {
+        file: 'files_campaign_file',
+        attachment: 'files_campaign_attachment',
+    },
+    template: {
+        file: 'files_template_file'
+    },
+    mosaico_template: {
+        file: 'files_mosaico_template_file',
+        block: 'files_mosaico_template_block'
+    }
+};
+
+exports.up = (knex, Promise) => (async() => {
+    await knex.schema.table('queued', table => {
+        table.integer('send_configuration').unsigned().notNullable();
+        table.integer('type').unsigned().notNullable(); // The values come from campaign-sender.js:MessageType
+        table.text('data', 'longtext');
+    });
+
+    const queued = await knex('queued')
+        .leftJoin('campaigns', 'queued.campaign', 'campaigns.id')
+        .select(['queued.id', 'queued.trigger', 'queued.campaign', 'campaigns.send_configuration']);
+
+    for (const queuedEntry of queued) {
+        const data = {};
+
+        if (queued.trigger) {
+            data.triggerId = queuedEntry.trigger;
+            data.campaignId = queuedEntry.campaign;
+        }
+
+        knex('queued')
+            .where('id', queuedEntry.id)
+            .update({
+                send_configuration: queuedEntry.send_configuration,
+                data: JSON.stringify(data)
+            });
+    }
+
+    await knex.schema.table('queued', table => {
+        table.dropColumn('trigger');
+        table.dropColumn('campaign');
+    });
+
+
+    for (const type in entityTypesWithFiles) {
+        const typeEntry = entityTypesWithFiles[type];
+
+        for (const subType in typeEntry) {
+            const subTypeEntry = typeEntry[subType];
+
+            await knex.schema.table(subTypeEntry, table => {
+                table.boolean('delete_pending').notNullable().defaultTo(false);
+                table.integer('lock_count').notNullable().defaultTo(0);
+            });
+        }
+    }
+})();
+
+exports.down = (knex, Promise) => (async() => {
+})();

server/setup/knex/migrations/20190616000000_drop_subject_in_send_configurations.js

@@ -0,0 +1,15 @@
+exports.up = (knex, Promise) => (async() => {
+    await knex.schema.table('send_configurations', table => {
+        table.dropColumn('subject');
+        table.dropColumn('subject_overridable');
+    });
+
+    await knex.schema.table('campaigns', table => {
+        table.renameColumn('subject_override', 'subject');
+    });
+
+    await knex('campaigns').whereNull('subject').update('subject', '');
+})();
+
+exports.down = (knex, Promise) => (async() => {
+})();

shared/activity-log.js

@@ -8,7 +8,8 @@ const EntityActivityType = {
 };
 
 const CampaignActivityType = {
-    STATUS_CHANGE: EntityActivityType.MAX + 1
+    STATUS_CHANGE: EntityActivityType.MAX + 1,
+    TEST_SEND: EntityActivityType.MAX + 2,
 };
 
 const ListActivityType = {

shared/campaigns.js

@@ -38,11 +38,12 @@ const CampaignStatus = {
 
     // For campaign types: NORMAL, RSS_ENTRY
     SENDING: 7,
+    PAUSING: 8,
 
-    MAX: 8
+    MAX: 9
 };
 
-const campaignOverridables = ['from_name', 'from_email', 'reply_to', 'subject'];
+const campaignOverridables = ['from_name', 'from_email', 'reply_to'];
 
 function getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration) {
     let allowedOverride = false;