diff --git a/client/src/account/API.js b/client/src/account/API.js index beb3f2bc..d3a07a4b 100644 --- a/client/src/account/API.js +++ b/client/src/account/API.js @@ -138,7 +138,7 @@ export default class API extends Component { {t('example')}
-curl -XPOST '{getUrl(`api/subscribe/B16uVTdW?access_token=${accessToken}`)}' \ +curl -XPOST '{getUrl(`api/subscribe/B16uVTdW?access_token=${accessToken}`)}' \
--data 'EMAIL=test@example.com&MERGE_CHECKBOX=yes&REQUIRE_CONFIRMATION=yes'POST /api/unsubscribe/:listId – {t('removeSubscription')}
@@ -165,7 +165,7 @@ export default class API extends Component { {t('example')} -curl -XPOST '{getUrl(`api/unsubscribe/B16uVTdW?access_token=${accessToken}`)}' \ +curl -XPOST '{getUrl(`api/unsubscribe/B16uVTdW?access_token=${accessToken}`)}' \
--data 'EMAIL=test@example.com'POST /api/delete/:listId – {t('deleteSubscription')}
@@ -192,7 +192,7 @@ export default class API extends Component { {t('example')} -curl -XPOST '{getUrl(`api/delete/B16uVTdW?access_token=${accessToken}`)}' \ +curl -XPOST '{getUrl(`api/delete/B16uVTdW?access_token=${accessToken}`)}' \
--data 'EMAIL=test@example.com'POST /api/field/:listId – {t('addNewCustomField')}
@@ -240,7 +240,7 @@ export default class API extends Component { {t('example')} -curl -XPOST '{getUrl(`api/field/B16uVTdW?access_token=${accessToken}`)}' \ +curl -XPOST '{getUrl(`api/field/B16uVTdW?access_token=${accessToken}`)}' \
--data 'NAME=Birthday&TYPE=birthday-us&VISIBLE=yes'GET /api/blacklist/get – {t('getListOfBlacklistedEmails')}
@@ -292,7 +292,7 @@ export default class API extends Component { {t('example')} -curl -XPOST '{getUrl(`api/blacklist/add?access_token={accessToken}`)}' \ +curl -XPOST '{getUrl(`api/blacklist/add?access_token={accessToken}`)}' \
--data 'EMAIL=test@example.com&'POST /api/blacklist/delete – {t('deleteEmailFromBlacklist')}
@@ -319,7 +319,7 @@ export default class API extends Component { {t('example')} -curl -XPOST '{getUrl(`api/blacklist/delete?access_token=${accessToken}`)}' \ +curl -XPOST '{getUrl(`api/blacklist/delete?access_token=${accessToken}`)}' \
--data 'EMAIL=test@example.com&'GET /api/lists/:email – {t('getTheListsAUserHasSubscribedTo')}
@@ -381,15 +381,14 @@ export default class API extends Component {
{t('example')}
-curl -XPOST '{getUrl(`api/templates/1/send?access_token={accessToken}`)}' \ +curl -XPOST '{getUrl(`api/templates/1/send?access_token=${accessToken}`)}' \); diff --git a/client/src/campaigns/CUD.js b/client/src/campaigns/CUD.js index 74585082..37a27284 100644 --- a/client/src/campaigns/CUD.js +++ b/client/src/campaigns/CUD.js @@ -244,7 +244,7 @@ export default class CUD extends Component { return filterData(data, [ 'name', 'description', 'segment', 'namespace', 'send_configuration', - 'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override', + 'from_name_override', 'from_email_override', 'reply_to_override', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url', 'type', 'source', 'parent', 'lists' ]); @@ -284,6 +284,8 @@ export default class CUD extends Component { send_configuration: null, namespace: mailtrainConfig.user.namespace, + subject: '', + click_tracking_disabled: false, open_tracking_disabled: false, @@ -326,6 +328,10 @@ export default class CUD extends Component { state.setIn(['name', 'error'], t('nameMustNotBeEmpty')); } + if (!state.getIn(['subject', 'value'])) { + state.setIn(['subject', 'error'], t('"Subject" line must not be empty"')); + } + if (!state.getIn(['send_configuration', 'value'])) { state.setIn(['send_configuration', 'error'], t('sendConfigurationMustBeSelected')); } @@ -592,7 +598,6 @@ export default class CUD extends Component { { data: 2, title: t('id'), render: data =>
--data 'EMAIL=test@example.com&SUBJECT=Test&VARIABLES[FOO]=bar&VARIABLES[TEST]=example'{data}
}, { data: 3, title: t('description') }, { data: 4, title: t('type'), render: data => this.mailerTypes[data].typeName }, - { data: 5, title: t('created'), render: data => moment(data).fromNow() }, { data: 6, title: t('namespace') } ]; @@ -604,10 +609,10 @@ export default class CUD extends Component { const addOverridable = (id, label) => { if(this.state.sendConfiguration[id + '_overridable']){ if (this.getFormValue(id + '_overriden')) { - sendSettings.push(); + sendSettings.push( ); } else { sendSettings.push( - + {this.state.sendConfiguration[id]} ); @@ -616,7 +621,7 @@ export default class CUD extends Component { } else{ sendSettings.push( -+ {this.state.sendConfiguration[id]} ); @@ -626,7 +631,8 @@ export default class CUD extends Component { addOverridable('from_name', t('fromName')); addOverridable('from_email', t('fromEmailAddress')); addOverridable('reply_to', t('replytoEmailAddress')); - addOverridable('subject', t('subjectLine')); + + sendSettings.push(); } else { sendSettings = {t('loadingSendConfiguration')} } @@ -760,8 +766,6 @@ export default class CUD extends Component { > } - - {templateEdit}diff --git a/client/src/campaigns/Content.js b/client/src/campaigns/Content.js index c1e42816..0a2da44c 100644 --- a/client/src/campaigns/Content.js +++ b/client/src/campaigns/Content.js @@ -20,7 +20,7 @@ import {getEditForm, getTemplateTypes, getTypeForm, ResourceType} from '../templ import axios from '../lib/axios'; import styles from "../lib/styles.scss"; import {getUrl} from "../lib/urls"; -import {TestSendModalDialog} from "./TestSendModalDialog"; +import {TestSendModalDialog, TestSendModalDialogMode} from "./TestSendModalDialog"; import {withComponentMixins} from "../lib/decorator-helpers"; import {ContentModalDialog} from "../lib/modals"; @@ -234,10 +234,11 @@ export default class CustomContent extends Component { return ( diff --git a/client/src/campaigns/Status.js b/client/src/campaigns/Status.js index 10d3bf34..5f70fc7f 100644 --- a/client/src/campaigns/Status.js +++ b/client/src/campaigns/Status.js @@ -16,6 +16,7 @@ import {CampaignStatus, CampaignType} from "../../../shared/campaigns"; import moment from 'moment'; import campaignsStyles from "./styles.scss"; import {withComponentMixins} from "../lib/decorator-helpers"; +import {TestSendModalDialog, TestSendModalDialogMode} from "./TestSendModalDialog"; @withComponentMixins([ @@ -25,7 +26,7 @@ import {withComponentMixins} from "../lib/decorator-helpers"; withPageHelpers, requiresAuthenticatedUser ]) -class TestUser extends Component { +class PreviewForTestUserModalDialog extends Component { constructor(props) { super(props); this.initForm({ @@ -34,7 +35,9 @@ class TestUser extends Component { } static propTypes = { - entity: PropTypes.object.isRequired + visible: PropTypes.bool.isRequired, + onHide: PropTypes.func.isRequired, + entity: PropTypes.object.isRequired, } localValidateFormValues(state) { @@ -64,6 +67,10 @@ class TestUser extends Component { } } + async hideModal() { + this.props.onHide(); + } + render() { const t = this.props.t; @@ -76,12 +83,14 @@ class TestUser extends Component { ]; return ( - +this.setState({showTestSendModal: false})} getDataAsync={this.sendModalGetDataHandler} - entity={this.props.entity} + campaign={this.props.entity} /> this.hideModal()} buttons={[ + { label: t('preview'), className: 'btn-primary', onClickAsync: ::this.previewAsync }, + { label: t('close'), className: 'btn-danger', onClickAsync: ::this.hideModal } + ]}> + + ); } } @@ -96,6 +105,12 @@ class TestUser extends Component { class SendControls extends Component { constructor(props) { super(props); + + this.state = { + showTestSendModal: false, + previewForTestUserVisible: false + }; + this.initForm({ leaveConfirmation: false }); @@ -257,13 +272,33 @@ class SendControls extends Component { const t = this.props.t; const entity = this.props.entity; - const yesNoDialog = ( -this.modalAction(false)} buttons={[ - { label: t('no'), className: 'btn-primary', onClickAsync: () => this.modalAction(false) }, - { label: t('yes'), className: 'btn-danger', onClickAsync: () => this.modalAction(true) } - ]}> - {this.state.modalMessage} - + const dialogs = ( + <> +this.setState({showTestSendModal: false})} + campaign={this.props.entity} + /> + this.setState({previewForTestUserVisible: false})} + entity={this.props.entity} + /> + this.modalAction(false)} buttons={[ + { label: t('no'), className: 'btn-primary', onClickAsync: () => this.modalAction(false) }, + { label: t('yes'), className: 'btn-danger', onClickAsync: () => this.modalAction(true) } + ]}> + {this.state.modalMessage} + + > + ); + + const testButtons = ( + <> +this.setState({previewForTestUserVisible: true})}/> + this.setState({showTestSendModal: true})}/> + > ); if (entity.status === CampaignStatus.IDLE || entity.status === CampaignStatus.PAUSED || (entity.status === CampaignStatus.SCHEDULED && entity.scheduled)) { @@ -271,7 +306,7 @@ class SendControls extends Component { const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`; return ( - {yesNoDialog} +{dialogs}+ ); + + } else if (entity.status === CampaignStatus.PAUSING) { + return ( +{entity.scheduled ? t('campaignIsScheduledForDelivery') : t('campaignIsReadyToBeSentOut')} @@ -292,20 +327,36 @@ class SendControls extends Component { :} + {entity.status === CampaignStatus.PAUSED && } {entity.status === CampaignStatus.PAUSED && } + {testButtons} + + {dialogs} +); } else if (entity.status === CampaignStatus.SENDING || (entity.status === CampaignStatus.SCHEDULED && !entity.scheduled)) { return ( -+ {t('Campaign is being paused. Please wait.')} + ++ + {testButtons} {yesNoDialog} +{dialogs}); @@ -314,7 +365,7 @@ class SendControls extends Component { const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`; return ( -{t('campaignIsBeingSentOut')} + {testButtons} {yesNoDialog} +{dialogs}); } else if (entity.status === CampaignStatus.INACTIVE) { return ( -{t('allMessagesSent!HitContinueIfYouYouWant')} @@ -322,35 +373,38 @@ class SendControls extends Component {+ {testButtons} {yesNoDialog} +{dialogs}); } else if (entity.status === CampaignStatus.ACTIVE) { return ( -{t('yourCampaignIsCurrentlyDisabledClick')} + {testButtons} {yesNoDialog} +{dialogs}); - } else { + } else { return null; } } @@ -441,7 +495,7 @@ export default class Status extends Component { addOverridable('from_name', t('fromName')); addOverridable('from_email', t('fromEmailAddress')); addOverridable('reply_to', t('replytoEmailAddress')); - addOverridable('subject', t('subjectLine')); + sendSettings.push({t('yourCampaignIsEnabledAndSendingMessages')} + {testButtons} {entity.subject} ); } else { sendSettings ={t('loadingSendConfiguration')} } @@ -491,13 +545,6 @@ export default class Status extends Component {- {(entity.type === CampaignType.REGULAR || entity.type === CampaignType.TRIGGERED) && -
-- } -
--
diff --git a/client/src/campaigns/TestSendModalDialog.js b/client/src/campaigns/TestSendModalDialog.js index c1f5d378..894c612e 100644 --- a/client/src/campaigns/TestSendModalDialog.js +++ b/client/src/campaigns/TestSendModalDialog.js @@ -5,13 +5,26 @@ import {withTranslation} from '../lib/i18n'; import PropTypes from 'prop-types'; import {ModalDialog} from "../lib/bootstrap-components"; import {requiresAuthenticatedUser, withPageHelpers} from "../lib/page"; -import {Form, TableSelect, withForm} from "../lib/form"; +import {CheckBox, Dropdown, Form, InputField, TableSelect, withForm} from "../lib/form"; import {withErrorHandling} from "../lib/error-handling"; import {getMailerTypes} from "../send-configurations/helpers"; import axios from '../lib/axios'; import {getUrl} from '../lib/urls'; import {withComponentMixins} from "../lib/decorator-helpers"; +import {CampaignType} from "../../../shared/campaigns"; +const Target = { + CAMPAIGN_ONE: 'campaign_one', + CAMPAIGN_ALL: 'campaign_all', + LIST_ONE: 'list_one', + LIST_ALL: 'list_all' +}; + +export const TestSendModalDialogMode = { + TEMPLATE: 0, + CAMPAIGN_CONTENT: 1, + CAMPAIGN_STATUS: 2 +} @withComponentMixins([ withTranslation, @@ -27,21 +40,39 @@ export class TestSendModalDialog extends Component { this.mailerTypes = getMailerTypes(props.t); this.initForm({ - leaveConfirmation: false + leaveConfirmation: false, + onChangeBeforeValidation: { + list: this.onListChanged + } }); } static propTypes = { - stateOwner: PropTypes.object, visible: PropTypes.bool.isRequired, + mode: PropTypes.number.isRequired, onHide: PropTypes.func.isRequired, - getDataAsync: PropTypes.func.isRequired, - entity: PropTypes.object + getDataAsync: PropTypes.func, + campaign: PropTypes.object + } + + onListChanged(mutStateData, key, oldValue, newValue) { + mutStateData.setIn(['segment', 'value'], null); } componentDidMount() { + const t = this.props.t; + this.populateFormValues({ - testUser: null, + target: Target.CAMPAIGN_ONE, + testUserSubscriptionCid: null, + testUserListAndSubscriptionCid: null, + subjectPrepend: '', + subjectAppend: t(' [Test]'), + sendConfiguration: null, + listCid: null, + list: null, + segment: null, + useSegmentation: false }); } @@ -52,25 +83,49 @@ export class TestSendModalDialog extends Component { async performAction() { const props = this.props; const t = props.t; + const mode = this.props.mode; if (this.isFormWithoutErrors()) { - try { this.hideFormValidation(); this.disableForm(); this.setFormStatusMessage('info', t('sendingTestEmail')); - const data = await this.props.getDataAsync(); + const data = {}; - const campaignCid = props.entity.cid; - const [listCid, subscriptionCid] = this.getFormValue('testUser').split(':'); + if (mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.TEMPLATE) { + const contentData = await this.props.getDataAsync(); + data.html = contentData.html; + data.text = contentData.text; + } - data.listCid = listCid; - data.subscriptionCid = subscriptionCid; - data.sendConfigurationId = props.entity.send_configuration; - data.campaignId = props.entity.id; + if (mode === TestSendModalDialogMode.TEMPLATE) { + data.listCid = this.getFormValue('listCid'); + data.subscriptionCid = this.getFormValue('testUserSubscriptionCid'); + data.sendConfigurationId = this.getFormValue('sendConfiguration'); - await axios.post(getUrl('rest/template-test-send'), data); + } else if (mode === TestSendModalDialogMode.CAMPAIGN_STATUS || mode === TestSendModalDialogMode.CAMPAIGN_CONTENT) { + data.campaignId = props.campaign.id; + data.subjectPrepend = this.getFormValue('subjectPrepend'); + data.subjectAppend = this.getFormValue('subjectAppend'); + + const target = this.getFormValue('target'); + if (target === Target.CAMPAIGN_ONE) { + const [listCid, subscriptionCid] = this.getFormValue('testUserListAndSubscriptionCid').split(':'); + data.listCid = listCid; + data.subscriptionCid = subscriptionCid; + + } else if (target === Target.LIST_ALL) { + data.listId = this.getFormValue('list'); + data.segmentId = this.getFormValue('useSegmentation') ? this.getFormValue('segment') : null; + + } else if (target === Target.LIST_ONE) { + data.listCid = this.getFormValue('listCid'); + data.subscriptionCid = this.getFormValue('testUserSubscriptionCid'); + } + } + + await axios.post(getUrl('rest/campaign-test-send'), data); this.clearFormStatusMessage(); @@ -87,32 +142,204 @@ export class TestSendModalDialog extends Component { localValidateFormValues(state) { const t = this.props.t; + const props = this.props; + const target = this.getFormValue('target'); + const mode = this.props.mode; - if (!state.getIn(['testUser', 'value'])) { - state.setIn(['testUser', 'error'], t('subscriptionHasToBeSelected')) - } else { - state.setIn(['testUser', 'error'], null); + state.setIn(['listCid', 'error'], null); + state.setIn(['sendConfiguration', 'error'], null); + state.setIn(['testUserSubscriptionCid', 'error'], null); + state.setIn(['testUserListAndSubscriptionCid', 'error'], null); + state.setIn(['list', 'error'], null); + state.setIn(['segment', 'error'], null); + + if (mode === TestSendModalDialogMode.TEMPLATE) { + if (!state.getIn(['listCid', 'value'])) { + state.setIn(['listCid', 'error'], t('listHasToBeSelected')) + } + + if (!state.getIn(['sendConfiguration', 'value'])) { + state.setIn(['sendConfiguration', 'error'], t('sendConfigurationHasToBeSelected')) + } + + if (!state.getIn(['testUserSubscriptionCid', 'value'])) { + state.setIn(['testUserSubscriptionCid', 'error'], t('subscriptionHasToBeSelected')) + } + } + + if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.CAMPAIGN_ONE) { + if (!state.getIn(['testUserListAndSubscriptionCid', 'value'])) { + state.setIn(['testUserListAndSubscriptionCid', 'error'], t('subscriptionHasToBeSelected')) + } + } + + if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ONE) { + if (!state.getIn(['listCid', 'value'])) { + state.setIn(['listCid', 'error'], t('listHasToBeSelected')) + } + + if (!state.getIn(['testUserSubscriptionCid', 'value'])) { + state.setIn(['testUserSubscriptionCid', 'error'], t('subscriptionHasToBeSelected')) + } + } + + if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ALL) { + if (!state.getIn(['list', 'value'])) { + state.setIn(['list', 'error'], t('listMustBeSelected')); + } + + if (state.getIn(['useSegmentation', 'value']) && !state.getIn(['segment', 'value'])) { + state.setIn(['segment', 'error'], t('segmentMustBeSelected')); + } } } render() { const t = this.props.t; + const props = this.props; - const testUsersColumns = [ - { data: 1, title: t('email') }, - { data: 2, title: t('subscriptionId'), render: data => {data}
}, - { data: 3, title: t('listId'), render: data =>{data}
}, - { data: 4, title: t('list') }, - { data: 5, title: t('listNamespace') } - ]; + const content = []; + const target = this.getFormValue('target'); + const mode = this.props.mode; + + if (mode === TestSendModalDialogMode.CAMPAIGN_STATUS) { + const targetOpts = [ + {key: Target.CAMPAIGN_ONE, label: t('Single test user of the campaign')}, + {key: Target.CAMPAIGN_ALL, label: t('All test users of the campaign')}, + {key: Target.LIST_ONE, label: t('Single test user from a list')}, + {key: Target.LIST_ALL, label: t('All test users from a list/segment')} + ]; + + content.push( ++ ); + } + + if (mode === TestSendModalDialogMode.TEMPLATE) { + const listCid = this.getFormValue('listCid'); + + const testUsersColumns = [ + { data: 1, title: t('subscriptionId'), render: data => {data}
}, + { data: 2, title: t('email') } + ]; + + const listsColumns = [ + { data: 1, title: t('name') }, + { data: 2, title: t('id'), render: data =>{data}
}, + { data: 3, title: t('subscribers') }, + { data: 4, title: t('description') }, + { data: 5, title: t('namespace') } + ]; + + const sendConfigurationsColumns = [ + { data: 1, title: t('name') }, + { data: 2, title: t('id'), render: data =>{data}
}, + { data: 3, title: t('description') }, + { data: 4, title: t('type'), render: data => this.mailerTypes[data].typeName }, + { data: 6, title: t('namespace') } + ]; + + content.push( ++ ); + + content.push( + + ); + + if (listCid) { + content.push( + + ); + } + } + + if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.CAMPAIGN_ONE) { + const testUsersColumns = [ + {data: 1, title: t('email')}, + {data: 2, title: t('subscriptionId'), render: data => {data}
}, + {data: 3, title: t('listId'), render: data =>{data}
}, + {data: 4, title: t('list')}, + {data: 5, title: t('listNamespace')} + ]; + + content.push( ++ ); + } + + if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ONE) { + const listCid = this.getFormValue('listCid'); + + const listsColumns = [ + { data: 1, title: t('name') }, + { data: 2, title: t('id'), render: data => {data}
}, + { data: 3, title: t('subscribers') }, + { data: 4, title: t('description') }, + { data: 5, title: t('namespace') } + ]; + + const testUsersColumns = [ + { data: 1, title: t('subscriptionId'), render: data =>{data}
}, + { data: 2, title: t('email') } + ]; + + content.push( ++ ); + + if (listCid) { + content.push( + + ); + } + } + + if ((mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) && target === Target.LIST_ALL) { + const listsColumns = [ + { data: 1, title: t('name') }, + { data: 2, title: t('id'), render: data => {data}
}, + { data: 3, title: t('subscribers') }, + { data: 4, title: t('description') }, + { data: 5, title: t('namespace') } + ]; + + const segmentsColumns = [ + { data: 1, title: t('name') } + ]; + + content.push( ++ ); + + const selectedList = this.getFormValue('list'); + content.push( + ++ ); + } + + if (mode === TestSendModalDialogMode.CAMPAIGN_CONTENT || mode === TestSendModalDialogMode.CAMPAIGN_STATUS) { + content.push( ++ {selectedList && this.getFormValue('useSegmentation') && + + } + + ); + + content.push( + + ); + } return ( this.hideModal()} buttons={[ { label: t('send'), className: 'btn-primary', onClickAsync: ::this.performAction }, - { label: t('cancel'), className: 'btn-danger', onClickAsync: ::this.hideModal } + { label: t('close'), className: 'btn-danger', onClickAsync: ::this.hideModal } ]}> ); diff --git a/client/src/campaigns/helpers.js b/client/src/campaigns/helpers.js index b0ac0be3..48bdbd4b 100644 --- a/client/src/campaigns/helpers.js +++ b/client/src/campaigns/helpers.js @@ -18,7 +18,8 @@ export function getCampaignLabels(t) { [CampaignStatus.PAUSED]: t('paused'), [CampaignStatus.INACTIVE]: t('inactive'), [CampaignStatus.ACTIVE]: t('active'), - [CampaignStatus.SENDING]: t('sending') + [CampaignStatus.SENDING]: t('sending'), + [CampaignStatus.PAUSING]: t('Pausing') }; diff --git a/client/src/lib/form.js b/client/src/lib/form.js index a0c5011e..73c9baf0 100644 --- a/client/src/lib/form.js +++ b/client/src/lib/form.js @@ -333,6 +333,13 @@ class InputField extends Component { const className = owner.addFormValidationClass('form-control', id); + /* This is for debugging purposes when React reports that InputField is uncontrolled + const value = owner.getFormValue(id); + if (value === null || value === undefined) console.log(`Warning: InputField ${id} is ${value}`); + */ + const value = owner.getFormValue(id); + if (value === null || value === undefined) console.log(`Warning: InputField ${id} is ${value}`); + return wrapInput(id, htmlId, owner, props.format, '', props.label, props.help, owner.updateFormValue(id, evt.target.value)}/> ); diff --git a/client/src/send-configurations/CUD.js b/client/src/send-configurations/CUD.js index 97687f13..074d9973 100644 --- a/client/src/send-configurations/CUD.js +++ b/client/src/send-configurations/CUD.js @@ -85,7 +85,7 @@ export default class CUD extends Component { } return filterData(data, ['name', 'description', 'from_email', 'from_email_overridable', 'from_name', - 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable', 'x_mailer', + 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'x_mailer', 'verp_hostname', 'verp_disable_sender_header', 'mailer_type', 'mailer_settings', 'namespace']); } @@ -103,8 +103,6 @@ export default class CUD extends Component { from_name_overridable: false, reply_to: '', reply_to_overridable: false, - subject: '', - subject_overridable: false, verpEnabled: false, verp_hostname: '', verp_disable_sender_header: false, @@ -233,8 +231,6 @@ export default class CUD extends Component {- - diff --git a/client/src/send-configurations/helpers.js b/client/src/send-configurations/helpers.js index ea9f2235..e78ccfbf 100644 --- a/client/src/send-configurations/helpers.js +++ b/client/src/send-configurations/helpers.js @@ -21,7 +21,7 @@ export function getMailerTypes(t) { const initVals = mailerTypes[mailerType].initData(); for (const key in initVals) { - if (!mutStateData.hasIn([key])) { + if (!mutStateData.hasIn([key, 'value']) || mutStateData.getIn([key, 'value']) === undefined) { mutStateData.setIn([key, 'value'], initVals[key]); } } diff --git a/client/src/templates/CUD.js b/client/src/templates/CUD.js index 7129b86b..dd9f5029 100644 --- a/client/src/templates/CUD.js +++ b/client/src/templates/CUD.js @@ -27,7 +27,7 @@ import {getEditForm, getTemplateTypes, getTypeForm} from './helpers'; import axios from '../lib/axios'; import styles from "../lib/styles.scss"; import {getUrl} from "../lib/urls"; -import {TestSendModalDialog} from "./TestSendModalDialog"; +import {TestSendModalDialog, TestSendModalDialogMode} from "../campaigns/TestSendModalDialog"; import {withComponentMixins} from "../lib/decorator-helpers"; import moment from 'moment'; @@ -298,6 +298,7 @@ export default class CUD extends Component { {isEdit &&this.setState({showTestSendModal: false})} getDataAsync={this.sendModalGetDataHandler}/> diff --git a/client/src/templates/TestSendModalDialog.js b/client/src/templates/TestSendModalDialog.js deleted file mode 100644 index 5d72d65a..00000000 --- a/client/src/templates/TestSendModalDialog.js +++ /dev/null @@ -1,149 +0,0 @@ -'use strict'; - -import React, {Component} from 'react'; -import {withTranslation} from '../lib/i18n'; -import PropTypes from 'prop-types'; -import {ModalDialog} from "../lib/bootstrap-components"; -import {requiresAuthenticatedUser, withPageHelpers} from "../lib/page"; -import {Form, TableSelect, withForm} from "../lib/form"; -import {withErrorHandling} from "../lib/error-handling"; -import moment from "moment"; -import {getMailerTypes} from "../send-configurations/helpers"; -import axios from '../lib/axios'; -import {getUrl} from "../lib/urls"; -import {withComponentMixins} from "../lib/decorator-helpers"; - - -@withComponentMixins([ - withTranslation, - withForm, - withErrorHandling, - withPageHelpers, - requiresAuthenticatedUser -]) -export class TestSendModalDialog extends Component { - constructor(props) { - super(props); - - this.mailerTypes = getMailerTypes(props.t); - - this.initForm({ - leaveConfirmation: false - }); - } - - static propTypes = { - stateOwner: PropTypes.object, - visible: PropTypes.bool.isRequired, - onHide: PropTypes.func.isRequired, - getDataAsync: PropTypes.func.isRequired - } - - componentDidMount() { - this.populateFormValues({ - list: null, - testUser: null, - sendConfiguration: null - }); - } - - async hideModal() { - this.props.onHide(); - } - - async performAction() { - const props = this.props; - const t = props.t; - - if (this.isFormWithoutErrors()) { - - try { - this.hideFormValidation(); - this.disableForm(); - this.setFormStatusMessage('info', t('sendingTestEmail')); - - const data = await this.props.getDataAsync(); - data.listCid = this.getFormValue('list'); - data.subscriptionCid = this.getFormValue('testUser'); - data.sendConfigurationId = this.getFormValue('sendConfiguration'); - - await axios.post(getUrl('rest/template-test-send'), data); - - this.clearFormStatusMessage(); - - this.enableForm(); - await this.hideModal(); - - } catch (err) { - throw err; - } - } else { - this.showFormValidation(); - } - } - - localValidateFormValues(state) { - const t = this.props.t; - - if (!state.getIn(['sendConfiguration', 'value'])) { - state.setIn(['sendConfiguration', 'error'], t('sendConfigurationHasToBeSelected')) - } else { - state.setIn(['sendConfiguration', 'error'], null); - } - - if (!state.getIn(['list', 'value'])) { - state.setIn(['list', 'error'], t('listHasToBeSelected')) - } else { - state.setIn(['list', 'error'], null); - } - - if (!state.getIn(['testUser', 'value'])) { - state.setIn(['testUser', 'error'], t('subscriptionHasToBeSelected')) - } else { - state.setIn(['testUser', 'error'], null); - } - } - - render() { - const t = this.props.t; - - const listId = this.getFormValue('list'); - - const testUsersColumns = [ - { data: 1, title: t('subscriptionId'), render: data => {data}
}, - { data: 2, title: t('email') } - ]; - - const listsColumns = [ - { data: 1, title: t('name') }, - { data: 2, title: t('id'), render: data =>{data}
}, - { data: 3, title: t('subscribers') }, - { data: 4, title: t('description') }, - { data: 5, title: t('namespace') } - ]; - - const sendConfigurationsColumns = [ - { data: 1, title: t('name') }, - { data: 2, title: t('id'), render: data =>{data}
}, - { data: 3, title: t('description') }, - { data: 4, title: t('type'), render: data => this.mailerTypes[data].typeName }, - { data: 5, title: t('created'), render: data => moment(data).fromNow() }, - { data: 6, title: t('namespace') } - ]; - - return ( -this.hideModal()} buttons={[ - { label: t('send'), className: 'btn-primary', onClickAsync: ::this.performAction }, - { label: t('cancel'), className: 'btn-danger', onClickAsync: ::this.hideModal } - ]}> - - - ); - } -} diff --git a/locales/en-US/common.json b/locales/en-US/common.json index dbc647da..467f1111 100644 --- a/locales/en-US/common.json +++ b/locales/en-US/common.json @@ -76,10 +76,10 @@ "forcesTheRssFeedCheckToImmediatelyCheck": "Forces the RSS feed check to immediately check the campaign with the given CID (in :campaignCid). It works only for RSS campaigns.", "sendTransactionalEmail": "Send transactional email", "sendSingleEmailByTemplateWithGiven": "Send single email by template with given templateId", - "idOfConfigurationUsedToCreateMailer": "ID of configuration used to create mailer instance", + "idOfConfigurationUsedToCreateMailer": "ID of configuration used to create mailer instance. If omitted, the default system send configuration is used.", "subject": "Subject", "dataPassedToTemplateWhenCompilingWith": "Data passed to template when compiling with Handlebars", - "mapOfTemplatesubjectVariablesToReplace": "Map of template/subject variables to replace", + "mapOfTemplatesubjectVariablesToReplace": "Map of template variables to replace", "apiResponseIsAJsonStructureWithErrorAnd": "API response is a JSON structure with <1>error1> and <3>data3> properties. If the response <5>error5> has a value set then the request failed.", "youNeedToDefineProperContentTypeWhen": "You need to define proper <1>Content-Type1> when making a request. You can either use <3>application/x-www-form-urlencoded3> for normal form data or <5>application/json5> for a JSON payload. Using <7>multipart/form-data7> is not supported.", "emailMustNotBeEmpty-1": "Email must not be empty", diff --git a/server/config/default.yaml b/server/config/default.yaml index 59800d5c..6b229ab9 100644 --- a/server/config/default.yaml +++ b/server/config/default.yaml @@ -260,9 +260,9 @@ roles: permissions: [view, edit, delete, share, createNamespace, createList, createCustomForm, createReport, createReportTemplate, createTemplate, createMosaicoTemplate, createSendConfiguration, createCampaign, manageUsers] children: sendConfiguration: [viewPublic, viewPrivate, edit, delete, share, sendWithoutOverrides, sendWithAllowedOverrides, sendWithAnyOverrides] - list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports] + list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports, send, sendToTestUsers] customForm: [view, edit, delete, share] - campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, fetchRss] + campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, sendToTestUsers, viewStats, fetchRss] template: [view, edit, delete, share, viewFiles, manageFiles] report: [view, edit, delete, share, execute, viewContent, viewOutput] reportTemplate: [view, edit, delete, share, execute] @@ -275,9 +275,9 @@ roles: permissions: [view, edit, delete, share, createNamespace, createList, createCustomForm, createReport, createTemplate, createMosaicoTemplate, createCampaign] children: sendConfiguration: [viewPublic, sendWithoutOverrides, sendWithAllowedOverrides] - list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports] + list: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports, send, sendToTestUsers] customForm: [view, edit, delete, share] - campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, fetchRss] + campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, sendToTestUsers, viewStats, fetchRss] template: [view, edit, delete, share, viewFiles, manageFiles] report: [view, edit, delete, share, execute, viewContent, viewOutput] reportTemplate: [view, share, execute] @@ -286,11 +286,11 @@ roles: campaignsCreator: name: Campaigns Creator - description: In the respective namespace, the user has all permissions for templates and campaigns. + description: In the respective namespace, the user has all permissions to create and manage templates and campaigns. permissions: [view, createTemplate, createCampaign] children: sendConfiguration: [viewPublic] - campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, fetchRss] + campaign: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, sendToTestUsers, fetchRss] template: [view, edit, delete, share, viewFiles, manageFiles] sendConfiguration: @@ -307,11 +307,11 @@ roles: master: name: Master description: All permissions - permissions: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports] + permissions: [view, edit, delete, share, viewFields, manageFields, viewSubscriptions, manageSubscriptions, viewSegments, manageSegments, viewImports, manageImports, send, sendToTestUsers] campaignsCreator: name: Campaigns Creator - description: The user can only use the list in setting up a campaign. However, this gives no permission to view subscriptions or to send to the list. - permissions: [view, viewFields, viewSegments] + description: The user can only use the list in setting up a campaign and to send email to test users. This gives no permission to view subscriptions or to send to the whole list. + permissions: [view, viewFields, viewSegments, sendToTestUsers] customForm: master: @@ -323,11 +323,11 @@ roles: master: name: Master description: All permissions - permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, viewStats, manageMessages, fetchRss] + permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, send, sendToTestUsers, viewStats, manageMessages, fetchRss] campaignsCreator: name: Campaigns Creator description: The user can setup the campaign but cannot send it. - permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, fetchRss] + permissions: [view, edit, delete, share, viewFiles, manageFiles, viewAttachments, manageAttachments, viewTriggers, manageTriggers, sendToTestUsers, fetchRss] rssTrigger: name: RSS Campaign Trigger description: Allows triggering a fetch of an RSS campaign diff --git a/server/lib/campaign-sender.js b/server/lib/campaign-sender.js index a0d7a44c..380b9bdc 100644 --- a/server/lib/campaign-sender.js +++ b/server/lib/campaign-sender.js @@ -21,215 +21,163 @@ const htmlToText = require('html-to-text'); const {getPublicUrl} = require('./urls'); const blacklist = require('../models/blacklist'); const libmime = require('libmime'); -const shares = require('../models/shares'); +const { enforce } = require('./helpers'); + +const MessageType = { + REGULAR: 0, + TRIGGERED: 1, + TEST: 2 +}; class CampaignSender { constructor() { } - static async testSend(context, listCid, subscriptionCid, campaignId, sendConfigurationId, html, text) { - let sendConfiguration, list, fieldsGrouped, campaign, subscriptionGrouped, useVerp, useVerpSenderHeader, mergeTags, attachments; - - await knex.transaction(async tx => { - sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), sendConfigurationId, false, true); - list = await lists.getByCidTx(tx, context, listCid); - fieldsGrouped = await fields.listGroupedTx(tx, list.id); - - useVerp = config.verp.enabled && sendConfiguration.verp_hostname; - useVerpSenderHeader = useVerp && !sendConfiguration.verp_disable_sender_header; - - subscriptionGrouped = await subscriptions.getByCid(context, list.id, subscriptionCid); - mergeTags = fields.getMergeTags(fieldsGrouped, subscriptionGrouped); - - if (campaignId) { - campaign = await campaigns.getByIdTx(tx, context, campaignId, false, campaigns.Content.WITHOUT_SOURCE_CUSTOM); - await campaigns.enforceSendPermissionTx(tx, context, campaign); - } else { - await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', sendConfigurationId, 'sendWithoutOverrides'); - - // This is to fake the campaign for getMessageLinks, which is called inside formatMessage - campaign = { - cid: '[CAMPAIGN_ID]' - }; - } - }); - - const encryptionKeys = []; - for (const fld of fieldsGrouped) { - if (fld.type === 'gpg' && mergeTags[fld.key]) { - encryptionKeys.push(mergeTags[fld.key].trim()); - } - } - - attachments = []; - // replace data: images with embedded attachments - html = html.replace(/(]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => { - const cid = shortid.generate() + '-attachments'; - attachments.push({ - path: dataUri, - cid - }); - return prefix + 'cid:' + cid; - }); - - html = tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, html, true); - - text = (text || '').trim() - ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, text) - : htmlToText.fromString(html, {wordwrap: 130}); - - - const mailer = await mailers.getOrCreateMailer(sendConfiguration.id); - - const getOverridable = key => { - return sendConfiguration[key]; - }; - - const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.'); - - const mail = { - from: { - name: getOverridable('from_name'), - address: getOverridable('from_email') - }, - replyTo: getOverridable('reply_to'), - xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false, - to: { - name: list.to_name === null ? undefined : tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false), - address: subscriptionGrouped.email - }, - sender: useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false, - - envelope: useVerp ? { - from: campaignAddress + '@' + sendConfiguration.verp_hostname, - to: subscriptionGrouped.email - } : false, - - headers: { - 'x-fbl': campaignAddress, - // custom header for SparkPost - 'x-msys-api': JSON.stringify({ - campaign_id: campaignAddress - }), - // custom header for SendGrid - 'x-smtpapi': JSON.stringify({ - unique_args: { - campaign_id: campaignAddress - } - }), - // custom header for Mailgun - 'x-mailgun-variables': JSON.stringify({ - campaign_id: campaignAddress - }), - 'List-ID': { - prepared: true, - value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>' - } - }, - list: { - unsubscribe: null - }, - subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false), - html, - text, - - attachments, - encryptionKeys - }; - - - let response; - try { - const info = await mailer.sendMassMail(mail); - response = info.response || info.messageId; - } catch (err) { - response = err.response || err.message; - } - - return response; - } - - - async init(settings) { + /* + settings is one of: + - campaignCid / campaignId + or + - sendConfiguration, listId, attachments, html, text, subject + */ + async _init(settings) { this.listsById = new Map(); // listId -> list this.listsByCid = new Map(); // listCid -> list this.listsFieldsGrouped = new Map(); // listId -> fieldsGrouped - this.attachments = []; await knex.transaction(async tx => { if (settings.campaignCid) { this.campaign = await campaigns.rawGetByTx(tx, 'cid', settings.campaignCid); - } else { + + } else if (settings.campaignId) { this.campaign = await campaigns.rawGetByTx(tx, 'id', settings.campaignId); + + } else { + // We are not within scope of a campaign (i.e. templates in MessageType.TEST message) + // This is to fake the campaign for getMessageLinks, which is called inside formatMessage + this.campaign = { + cid: '[CAMPAIGN_ID]', + from_name_override: null, + from_email_override: null, + reply_to_override: null + }; } - const campaign = this.campaign; - - this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration, false, true); - - for (const listSpec of campaign.lists) { - const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list); - this.listsById.set(list.id, list); - this.listsByCid.set(list.cid, list); - this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id)); - } - - if (campaign.source === CampaignSource.TEMPLATE) { - this.template = await templates.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.data.sourceTemplate, false); - } - - const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', campaign.id); - for (const attachment of attachments) { - this.attachments.push({ - filename: attachment.originalname, - path: files.getFilePath('campaign', 'attachment', campaign.id, attachment.filename) - }); + if (settings.sendConfigurationId) { + this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), settings.sendConfigurationId, false, true); + } else if (this.campaign.send_configuration) { + this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.send_configuration, false, true); + } else { + enforce(false); } this.useVerp = config.verp.enabled && this.sendConfiguration.verp_hostname; this.useVerpSenderHeader = this.useVerp && !this.sendConfiguration.verp_disable_sender_header; + + if (settings.listId) { + const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), settings.listId); + this.listsById.set(list.id, list); + this.listsByCid.set(list.cid, list); + this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id)); + + } else if (this.campaign.lists) { + for (const listSpec of this.campaign.lists) { + const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list); + this.listsById.set(list.id, list); + this.listsByCid.set(list.cid, list); + this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id)); + } + + } else { + enforce(false); + } + + if (settings.attachments) { + this.attachments = settings.attachments; + + } else if (this.campaign.id) { + const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', this.campaign.id); + + this.attachments = []; + for (const attachment of attachments) { + this.attachments.push({ + filename: attachment.originalname, + path: files.getFilePath('campaign', 'attachment', this.campaign.id, attachment.filename) + }); + } + + } else { + this.attachments = []; + } + + if (settings.html !== undefined) { + this.html = settings.html; + this.text = settings.text; + } else if (this.campaign.source === CampaignSource.TEMPLATE) { + this.template = await templates.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.data.sourceTemplate, false); + } + + if (settings.subject !== undefined) { + this.subject = settings.subject; + } else if (this.campaign.subject !== undefined) { + this.subject = this.campaign.subject; + } else { + enforce(false); + } }); } - async _getMessage(campaign, list, subscriptionGrouped, mergeTags, replaceDataImgs) { + async _getMessage(list, subscriptionGrouped, mergeTags, replaceDataImgs) { let html = ''; let text = ''; let renderTags = false; + const campaign = this.campaign; - if (campaign.source === CampaignSource.URL) { - const form = tools.getMessageLinks(campaign, list, subscriptionGrouped); - for (const key in mergeTags) { - form[key] = mergeTags[key]; - } - - const response = await request.post({ - uri: campaign.sourceUrl, - form, - resolveWithFullResponse: true - }); - - if (response.statusCode !== 200) { - throw new Error(`Received status code ${httpResponse.statusCode} from ${campaign.sourceUrl}`); - } - - html = response.body; - text = ''; + if (this.renderedHtml !== undefined) { + html = this.renderedHtml; + text = this.renderedText; renderTags = false; - } else if (campaign.source === CampaignSource.CUSTOM || campaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN || campaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE) { - html = campaign.data.sourceCustom.html; - text = campaign.data.sourceCustom.text; + } else if (this.html !== undefined) { + html = this.html; + text = this.text; renderTags = true; - } else if (campaign.source === CampaignSource.TEMPLATE) { - const template = this.template; - html = template.html; - text = template.text; - renderTags = true; + } else { + if (campaign.source === CampaignSource.URL) { + const form = tools.getMessageLinks(campaign, list, subscriptionGrouped); + for (const key in mergeTags) { + form[key] = mergeTags[key]; + } + + const response = await request.post({ + uri: campaign.sourceUrl, + form, + resolveWithFullResponse: true + }); + + if (response.statusCode !== 200) { + throw new Error(`Received status code ${httpResponse.statusCode} from ${campaign.sourceUrl}`); + } + + html = response.body; + text = ''; + renderTags = false; + + } else if (campaign.source === CampaignSource.CUSTOM || campaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN || campaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE) { + html = campaign.data.sourceCustom.html; + text = campaign.data.sourceCustom.text; + renderTags = true; + + } else if (campaign.source === CampaignSource.TEMPLATE) { + const template = this.template; + html = template.html; + text = template.text; + renderTags = true; + } + + html = await links.updateLinks(campaign, list, subscriptionGrouped, mergeTags, html); } - html = await links.updateLinks(campaign, list, subscriptionGrouped, mergeTags, html); - const attachments = this.attachments.slice(); if (replaceDataImgs) { // replace data: images with embedded attachments @@ -273,6 +221,14 @@ class CampaignSender { return tags; } + async initByCampaignCid(campaignCid) { + await this._init({campaignCid}); + } + + async initByCampaignId(campaignId) { + await this._init({campaignId}); + } + async getMessage(listCid, subscriptionCid) { const list = this.listsByCid.get(listCid); const subscriptionGrouped = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid); @@ -280,20 +236,36 @@ class CampaignSender { const campaign = this.campaign; const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign)); - return await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, false); + return await this._getMessage(list, subscriptionGrouped, mergeTags, false); } - async sendMessageByEmail(listId, email) { - const subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, email); - await this._sendMessage(listId, subscriptionGrouped); - } - async sendMessageBySubscriptionId(listId, subscriptionId) { - const subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, subscriptionId); - await this._sendMessage(listId, subscriptionGrouped); - } + /* + subData is one of: + - queuedMessage + or + - email, listId + */ + async _sendMessage(subData) { + let msgType; + let subscriptionGrouped; + let listId; + + if (subData.queuedMessage) { + const queuedMessage = subData.queuedMessage; + msgType = queuedMessage.type; + listId = queuedMessage.list; + subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, queuedMessage.subscription); + + } else { + enforce(subData.email); + enforce(subData.listId); + + msgType = MessageType.REGULAR; + listId = subData.listId; + subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, subData.email); + } - async _sendMessage(listId, subscriptionGrouped) { const email = subscriptionGrouped.email; if (await blacklist.isBlacklisted(email)) { @@ -315,7 +287,7 @@ class CampaignSender { const sendConfiguration = this.sendConfiguration; - const {html, text, attachments} = await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, true); + const {html, text, attachments} = await this._getMessage(list, subscriptionGrouped, mergeTags, true); const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.'); @@ -380,7 +352,7 @@ class CampaignSender { list: { unsubscribe: listUnsubscribe }, - subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false), + subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, this.subject, false), html, text, @@ -435,17 +407,47 @@ class CampaignSender { } - await knex('campaigns').where('id', campaign.id).increment('delivered'); + if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) { + await knex('campaigns').where('id', campaign.id).increment('delivered'); + } } catch (err) { + console.log(err); + + /* + { Error: connect ECONNREFUSED 127.0.0.1:55871 + at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14) + cause: + { Error: connect ECONNREFUSED 127.0.0.1:55871 + at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14) + stack: + 'Error: connect ECONNREFUSED 127.0.0.1:55871\n at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)', + errno: 'ECONNREFUSED', + code: 'ECONNECTION', + syscall: 'connect', + address: '127.0.0.1', + port: 55871, + command: 'CONN' }, + isOperational: true, + errno: 'ECONNREFUSED', + code: 'ECONNECTION', + syscall: 'connect', + address: '127.0.0.1', + port: 55871, + command: 'CONN' } + + */ + status = SubscriptionStatus.BOUNCED; response = err.response || err.message; - await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced'); + if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) { + await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced'); + } } const now = new Date(); - if (campaign.type === CampaignType.REGULAR || campaign.type === CampaignType.RSS_ENTRY) { + if (msgType === MessageType.REGULAR) { await knex('campaign_messages').insert({ campaign: this.campaign.id, list: list.id, @@ -457,16 +459,64 @@ class CampaignSender { updated: now }); - } else if (campaign.type = CampaignType.TRIGGERED) { + } else if (msgType === MessageType.TRIGGERED || msgType === MessageType.TEST) { + if (subData.queuedMessage.data.attachments) { + for (const attachment of subData.queuedMessage.data.attachments) { + try { + // We ignore any errors here because we already sent the message. Thus we have to mark it as completed to avoid sending it again. + await knex.transaction(async tx => { + await files.unlockTx(tx, 'campaign', 'attachment', attachment.id); + }); + } catch (err) { + log.error('CampaignSender', `Error when unlocking attachment ${attachment.id} for ${listId}:${subscriptionGrouped.email} (queuedId: ${subData.queuedMessage.id})`); + log.verbose(err.stack); + } + } + } + await knex('queued') - .where({ - campaign: this.campaign.id, - list: list.id, - subscription: subscriptionGrouped.id - }) + .where({id: subData.queuedMessage.id}) .del(); } } + + async sendRegularMessage(listId, email) { + await this._sendMessage({listId, email}); + } } -module.exports = CampaignSender; \ No newline at end of file +CampaignSender.sendQueuedMessage = async (queuedMessage) => { + const msgData = queuedMessage.data; + + const cs = new CampaignSender(); + await cs._init({ + campaignId: msgData.campaignId, + listId: queuedMessage.list, + sendConfigurationId: queuedMessage.send_configuration, + attachments: msgData.attachments, + html: msgData.html, + text: msgData.text, + subject: msgData.subject + }); + + await cs._sendMessage({queuedMessage}); +}; + +CampaignSender.queueMessageTx = async (tx, sendConfigurationId, listId, subscriptionId, messageType, messageData) => { + if (messageData.attachments) { + for (const attachment of messageData.attachments) { + await files.lockTx(tx,'campaign', 'attachment', attachment.id); + } + } + + await tx('queued').insert({ + send_configuration: sendConfigurationId, + list: listId, + subscription: subscriptionId, + type: messageType, + data: JSON.stringify(messageData) + }); +}; + +module.exports.CampaignSender = CampaignSender; +module.exports.MessageType = MessageType; \ No newline at end of file diff --git a/server/lib/dependency-helpers.js b/server/lib/dependency-helpers.js index 03ff1e39..02354966 100644 --- a/server/lib/dependency-helpers.js +++ b/server/lib/dependency-helpers.js @@ -4,7 +4,6 @@ const knex = require('./knex'); const interoperableErrors = require('../../shared/interoperable-errors'); const entitySettings = require('./entity-settings'); const shares = require('../models/shares'); -const { enforce } = require('./helpers'); const defaultNoOfDependenciesReported = 20; @@ -21,7 +20,7 @@ async function ensureNoDependencies(tx, context, id, depSpecs) { if (depSpec.query) { rows = await depSpec.query(tx).limit(defaultNoOfDependenciesReported + 1); } else if (depSpec.column) { - rows = await tx(entityType.entitiesTable).where(depSpec.column, id).select(['id', 'name']).limit(defaultNoOfDependenciesReported + 1); + rows = await tx(entityType.entitiesTable).where(depSpec.column, id).forShare().select(['id', 'name']).limit(defaultNoOfDependenciesReported + 1); } else if (depSpec.rows) { rows = await depSpec.rows(tx, defaultNoOfDependenciesReported + 1) } diff --git a/server/lib/entity-settings.js b/server/lib/entity-settings.js index cf23e724..3bb34927 100644 --- a/server/lib/entity-settings.js +++ b/server/lib/entity-settings.js @@ -44,6 +44,7 @@ const entityTypes = { }, attachment: { table: 'files_campaign_attachment', + inUseTable: 'files_campaign_attachment_usage', permissions: { view: 'viewAttachments', manage: 'manageAttachments' diff --git a/server/lib/helpers.js b/server/lib/helpers.js index 2f87df96..de5a7baa 100644 --- a/server/lib/helpers.js +++ b/server/lib/helpers.js @@ -28,11 +28,11 @@ function filterObject(obj, allowedKeys) { return result; } -function castToInteger(id) { +function castToInteger(id, msg) { const val = parseInt(id); if (!Number.isInteger(val)) { - throw new Error('Invalid id'); + throw new Error(msg || 'Invalid id'); } return val; diff --git a/server/lib/mailers.js b/server/lib/mailers.js index 7880ce1a..e9faf41a 100644 --- a/server/lib/mailers.js +++ b/server/lib/mailers.js @@ -94,14 +94,18 @@ async function _sendMail(transport, mail, template) { return await trySendAsync(); } -async function _sendTransactionalMail(transport, mail, template) { - const sendConfiguration = transport.mailer.sendConfiguration; - +async function _sendTransactionalMail(transport, mail) { if (!mail.headers) { mail.headers = {}; } mail.headers['X-Sending-Zone'] = 'transactional'; + return await _sendMail(transport, mail); +} + +async function _sendTransactionalMailBasedOnTemplate(transport, mail, template) { + const sendConfiguration = transport.mailer.sendConfiguration; + mail.from = { name: sendConfiguration.from_name, address: sendConfiguration.from_email @@ -129,7 +133,7 @@ async function _sendTransactionalMail(transport, mail, template) { }); } - return await _sendMail(transport, mail); + return await _sendTransactionalMail(transport, mail); } async function _createTransport(sendConfiguration) { @@ -263,7 +267,8 @@ async function _createTransport(sendConfiguration) { transport.mailer = { sendConfiguration, throttleWait: bluebird.promisify(throttleWait), - sendTransactionalMail: async (mail, template) => await _sendTransactionalMail(transport, mail, template), + sendTransactionalMail: async (mail, template) => await _sendTransactionalMail(transport, mail), + sendTransactionalMailBasedOnTemplate: async (mail, template) => await _sendTransactionalMailBasedOnTemplate(transport, mail, template), sendMassMail: async (mail, template) => await _sendMail(transport, mail) }; diff --git a/server/lib/subscription-mail-helpers.js b/server/lib/subscription-mail-helpers.js index ad7d0202..838cb75c 100644 --- a/server/lib/subscription-mail-helpers.js +++ b/server/lib/subscription-mail-helpers.js @@ -139,7 +139,7 @@ async function _sendMail(list, email, template, locale, subjectKey, relativeUrls try { if (list.send_configuration) { const mailer = await mailers.getOrCreateMailer(list.send_configuration); - await mailer.sendTransactionalMail({ + await mailer.sendTransactionalMailBasedOnTemplate({ to: { name: getDisplayName(flds, subscription), address: email diff --git a/server/lib/template-sender.js b/server/lib/template-sender.js deleted file mode 100644 index 534df7fe..00000000 --- a/server/lib/template-sender.js +++ /dev/null @@ -1,87 +0,0 @@ -'use strict'; - -const mailers = require('./mailers'); -const tools = require('./tools'); -const templates = require('../models/templates'); -const { getMergeTagsForBases } = require('../../shared/templates'); -const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls'); - -class TemplateSender { - constructor(options) { - this.defaultOptions = { - maxMails: 100, - ...options - }; - } - - async send(params) { - const options = { ...this.defaultOptions, ...params }; - this._validateMailOptions(options); - - const [mailer, template] = await Promise.all([ - mailers.getOrCreateMailer( - options.sendConfigurationId - ), - templates.getById( - options.context, - options.templateId, - false - ) - ]); - - const variables = { - EMAIL: options.email, - ...getMergeTagsForBases(getTrustedUrl(), getSandboxUrl(), getPublicUrl()), - ...options.variables - }; - - const html = tools.formatTemplate( - template.html, - null, - variables, - true - ); - const subject = tools.formatTemplate( - options.subject || template.description || template.name, - variables - ); - return mailer.sendTransactionalMail( - { - to: options.email, - subject - }, - { - html: { template: html }, - data: options.data, - locale: options.locale - } - ); - } - - _validateMailOptions(options) { - let { context, email, locale, templateId } = options; - - if (!templateId) { - throw new Error('Missing templateId'); - } - if (!context) { - throw new Error('Missing context'); - } - if (!email || email.length === 0) { - throw new Error('Missing email'); - } - if (typeof email === 'string') { - email = email.split(','); - } - if (email.length > options.maxMails) { - throw new Error( - `Cannot send more than ${options.maxMails} emails at once` - ); - } - if (!locale) { - throw new Error('Missing locale'); - } - } -} - -module.exports = TemplateSender; diff --git a/server/models/blacklist.js b/server/models/blacklist.js index 95f92460..8dca9c17 100644 --- a/server/models/blacklist.js +++ b/server/models/blacklist.js @@ -41,16 +41,17 @@ async function search(context, offset, limit, search) { async function add(context, email) { enforce(email, 'Email has to be set'); - return await knex.transaction(async tx => { - shares.enforceGlobalPermission(context, 'manageBlacklist'); - - const existing = await tx('blacklist').where('email', email).first(); - if (!existing) { - await tx('blacklist').insert({email}); - } + shares.enforceGlobalPermission(context, 'manageBlacklist'); + try { + await knex('blacklist').insert({email}); await activityLog.logBlacklistActivity(BlacklistActivityType.ADD, email); - }); + } catch (err) { + if (err.code === 'ER_DUP_ENTRY') { + } else { + throw err; + } + } } async function remove(context, email) { diff --git a/server/models/campaigns.js b/server/models/campaigns.js index 483ee32f..98a02592 100644 --- a/server/models/campaigns.js +++ b/server/models/campaigns.js @@ -21,12 +21,14 @@ const {LinkId} = require('./links'); const feedcheck = require('../lib/feedcheck'); const contextHelpers = require('../lib/context-helpers'); const {convertFileURLs} = require('../lib/campaign-content'); +const {CampaignSender, MessageType} = require('../lib/campaign-sender'); +const lists = require('./lists'); const {EntityActivityType, CampaignActivityType} = require('../../shared/activity-log'); const activityLog = require('../lib/activity-log'); const allowedKeysCommon = ['name', 'description', 'segment', 'namespace', - 'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url']; + 'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url']; const allowedKeysCreate = new Set(['type', 'source', ...allowedKeysCommon]); const allowedKeysCreateRssEntry = new Set(['type', 'source', 'parent', ...allowedKeysCommon]); @@ -168,7 +170,7 @@ async function listTestUsersDTAjax(context, campaignId, params) { let subsQry; if (subsQrys.length === 1) { - const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`' + const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`'; subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings); } else { @@ -342,7 +344,7 @@ async function rawGetByTx(tx, key, id) { .groupBy('campaigns.id') .select([ 'campaigns.id', 'campaigns.cid', 'campaigns.name', 'campaigns.description', 'campaigns.namespace', 'campaigns.status', 'campaigns.type', 'campaigns.source', - 'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject_override', + 'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject', 'campaigns.data', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled', 'campaigns.unsubscribe_url', 'campaigns.scheduled', 'campaigns.delivered', 'campaigns.unsubscribed', 'campaigns.bounced', 'campaigns.complained', 'campaigns.blacklisted', 'campaigns.opened', 'campaigns.clicks', knex.raw(`GROUP_CONCAT(CONCAT_WS(\':\', campaign_lists.list, campaign_lists.segment) ORDER BY campaign_lists.id SEPARATOR \';\') as lists`) @@ -632,21 +634,32 @@ async function remove(context, id) { }); } -async function enforceSendPermissionTx(tx, context, campaignId) { +async function enforceSendPermissionTx(tx, context, campaignOrCampaignId, isToTestUsers, listId) { let campaign; - if (typeof campaignId === 'object') { - campaign = campaignId; + if (typeof campaignOrCampaignId === 'object') { + campaign = campaignOrCampaignId; } else { - campaign = await getByIdTx(tx, context, campaignId, false); + campaign = await getByIdTx(tx, context, campaignOrCampaignId, false); } const sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration, false, false); - const requiredPermission = getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration); + const requiredSendConfigurationPermission = getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration); + await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', campaign.send_configuration, requiredSendConfigurationPermission); - await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', campaign.send_configuration, requiredPermission); - await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaign.id, 'send'); + const requiredListAndCampaignPermission = isToTestUsers ? 'sendToTestUsers' : 'send'; + + await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaign.id, requiredListAndCampaignPermission); + + if (listId) { + await shares.enforceEntityPermissionTx(tx, context, 'list', listId, requiredListAndCampaignPermission); + + } else { + for (const listIds of campaign.lists) { + await shares.enforceEntityPermissionTx(tx, context, 'list', listIds.list, requiredListAndCampaignPermission); + } + } } @@ -845,12 +858,9 @@ async function getSubscribersQueryGeneratorTx(tx, campaignId) { async function _changeStatus(context, campaignId, permittedCurrentStates, newState, invalidStateMessage, scheduled = null) { await knex.transaction(async tx => { - const entity = await tx('campaigns').where('id', campaignId).first(); - if (!entity) { - throw new interoperableErrors.NotFoundError(); - } + const entity = await getByIdTx(tx, context, campaignId, false); - await enforceSendPermissionTx(tx, context, entity); + await enforceSendPermissionTx(tx, context, entity, false); if (!permittedCurrentStates.includes(entity.status)) { throw new interoperableErrors.InvalidStateError(invalidStateMessage); @@ -869,11 +879,11 @@ async function _changeStatus(context, campaignId, permittedCurrentStates, newSta async function start(context, campaignId, startAt) { - await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.SCHEDULED, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt); + await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE, PAUSED, or FINISHED state', startAt); } async function stop(context, campaignId) { - await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED], CampaignStatus.PAUSED, 'Cannot stop campaign until it is in SCHEDULED state'); + await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED, CampaignStatus.SENDING], CampaignStatus.PAUSING, 'Cannot stop campaign until it is in SCHEDULED or SENDING state'); } async function reset(context, campaignId) { @@ -944,6 +954,103 @@ async function fetchRssCampaign(context, cid) { }); } +async function testSend(context, data) { + // Though it's a bit counterintuitive, this handles also test sends of a template (i.e. without any campaign id) + + await knex.transaction(async tx => { + const processSubscriber = async (sendConfigurationId, listId, subscriptionId, messageData) => { + await CampaignSender.queueMessageTx(tx, sendConfigurationId, listId, subscriptionId, MessageType.TEST, messageData); + + await activityLog.logEntityActivity('campaign', CampaignActivityType.TEST_SEND, campaignId, {list: listId, subscription: subscriptionId}); + }; + + const campaignId = data.campaignId; + + if (campaignId) { // This means we are sending a campaign + /* + Data coming from the client: + - html, text + - subjectPrepend, subjectAppend + - listCid, subscriptionCid + - listId, segmentId + */ + + const campaign = await getByIdTx(tx, context, campaignId, false); + const sendConfigurationId = campaign.send_configuration; + + const messageData = { + campaignId: campaignId, + subject: data.subjectPrepend + campaign.subject + data.subjectAppend, + html: data.html, // The html and text may be undefined + text: data.text, + attachments: [] + }; + + const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', campaignId); + for (const attachment of attachments) { + messageData.attachments.push({ + filename: attachment.originalname, + path: files.getFilePath('campaign', 'attachment', campaign.id, attachment.filename), + id: attachment.id + }); + } + + + let listId = data.listId; + if (!listId && data.listCid) { + const list = await lists.getByCidTx(tx, context, data.listCid); + listId = list.id; + } + + const segmentId = data.segmentId; + + if (listId) { + await enforceSendPermissionTx(tx, context, campaign, true, listId); + + if (data.subscriptionCid) { + const subscriber = await subscriptions.getByCidTx(tx, context, listId, data.subscriptionCid); + await processSubscriber(sendConfigurationId, listId, subscriber.id, messageData); + + } else { + const subscribers = await subscriptions.listTestUsersTx(tx, context, listId, segmentId); + for (const subscriber of subscribers) { + await processSubscriber(sendConfigurationId, listId, subscriber.id, messageData); + } + } + + } else { + for (const lstSeg of campaign.lists) { + await enforceSendPermissionTx(tx, context, campaign, true, lstSeg.list); + + const subscribers = await subscriptions.listTestUsersTx(tx, context, lstSeg.list, segmentId); + for (const subscriber of subscribers) { + await processSubscriber(sendConfigurationId, lstSeg.list, subscriber.id, messageData); + } + } + } + + } else { // This means we are sending a template + /* + Data coming from the client: + - html, text + - listCid, subscriptionCid, sendConfigurationId + */ + + const messageData = { + subject: 'Test', + html: data.html, + text: data.text + }; + + const list = await lists.getByCidTx(tx, context, data.listCid); + const subscriber = await subscriptions.getByCidTx(tx, context, list.id, data.subscriptionCid); + await processSubscriber(data.sendConfigurationId, list.id, subscriber.id, messageData); + } + }); + + senders.scheduleCheck(); +} + module.exports.Content = Content; module.exports.hash = hash; @@ -986,4 +1093,6 @@ module.exports.rawGetByTx = rawGetByTx; module.exports.getTrackingSettingsByCidTx = getTrackingSettingsByCidTx; module.exports.getStatisticsOpened = getStatisticsOpened; -module.exports.fetchRssCampaign = fetchRssCampaign; \ No newline at end of file +module.exports.fetchRssCampaign = fetchRssCampaign; + +module.exports.testSend = testSend; \ No newline at end of file diff --git a/server/models/files.js b/server/models/files.js index 2ceaf928..01afe512 100644 --- a/server/models/files.js +++ b/server/models/files.js @@ -45,7 +45,7 @@ async function listDTAjax(context, type, subType, entityId, params) { await shares.enforceEntityPermission(context, type, entityId, getFilesPermission(type, subType, 'view')); return await dtHelpers.ajaxList( params, - builder => builder.from(getFilesTable(type, subType)).where({entity: entityId}), + builder => builder.from(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false}), ['id', 'originalname', 'filename', 'size', 'created'] ); } @@ -53,7 +53,7 @@ async function listDTAjax(context, type, subType, entityId, params) { async function listTx(tx, context, type, subType, entityId) { enforceTypePermitted(type, subType); await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view')); - return await tx(getFilesTable(type, subType)).where({entity: entityId}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc'); + return await tx(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc'); } async function list(context, type, subType, entityId) { @@ -65,7 +65,7 @@ async function list(context, type, subType, entityId) { async function getFileById(context, type, subType, id) { enforceTypePermitted(type, subType); const file = await knex.transaction(async tx => { - const file = await tx(getFilesTable(type, subType)).where('id', id).first(); + const file = await tx(getFilesTable(type, subType)).where({id: id, delete_pending: false}).first(); await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'view')); return file; }); @@ -85,7 +85,7 @@ async function _getFileBy(context, type, subType, entityId, key, value) { enforceTypePermitted(type, subType); const file = await knex.transaction(async tx => { await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view')); - const file = await tx(getFilesTable(type, subType)).where({entity: entityId, [key]: value}).first(); + const file = await tx(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false, [key]: value}).first(); return file; }); @@ -155,7 +155,7 @@ async function createFiles(context, type, subType, entityId, files, replacementB await knex.transaction(async tx => { await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage')); - const existingNamesRows = await tx(getFilesTable(type, subType)).where('entity', entityId).select(['id', 'filename', 'originalname']); + const existingNamesRows = await tx(getFilesTable(type, subType)).where({entity: entityId, delete_pending: false}).select(['id', 'filename', 'originalname']); const existingNameSet = new Set(); for (const row of existingNamesRows) { @@ -275,18 +275,49 @@ async function createFiles(context, type, subType, entityId, files, replacementB } } +async function lockTx(tx, type, subType, id) { + enforceTypePermitted(type, subType); + const filesTableName = getFilesTable(type, subType); + await tx(filesTableName).where('id', id).increment('lock_count'); +} + +async function unlockTx(tx, type, subType, id) { + enforceTypePermitted(type, subType); + + const filesTableName = getFilesTable(type, subType); + const file = await tx(filesTableName).where('id', id).first(); + + enforce(file, `File ${id} not found`); + enforce(file.lock_count > 0, `Corrupted lock count at file ${id}`); + + if (file.lock_count === 1 && file.delete_pending) { + await tx(filesTableName).where('id', id).del(); + + const filePath = getFilePath(type, subType, file.entity, file.filename); + await fs.removeAsync(filePath); + + } else { + await tx(filesTableName).where('id', id).update({lock_count: file.lock_count - 1}); + } +} + async function removeFile(context, type, subType, id) { enforceTypePermitted(type, subType); - const file = await knex.transaction(async tx => { - const file = await tx(getFilesTable(type, subType)).where('id', id).select('entity', 'filename').first(); + await knex.transaction(async tx => { + const filesTableName = getFilesTable(type, subType); + const file = await tx(filesTableName).where('id', id).first(); await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'manage')); - await tx(getFilesTable(type, subType)).where('id', id).del(); - return {filename: file.filename, entity: file.entity}; - }); - const filePath = getFilePath(type, subType, file.entity, file.filename); - await fs.removeAsync(filePath); + if (!file.lock_count) { + await tx(filesTableName).where('id', file.id).del(); + + const filePath = getFilePath(type, subType, file.entity, file.filename); + await fs.removeAsync(filePath); + } else { + await tx(filesTableName).where('id', file.id).update({delete_pending: true}); + } + }); } async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toType, toSubType, toEntityId) { @@ -296,7 +327,7 @@ async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toTyp enforceTypePermitted(toType, toSubType); await shares.enforceEntityPermissionTx(tx, context, toType, toEntityId, getFilesPermission(toType, toSubType, 'manage')); - const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId}); + const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId, delete_pending: false}); for (const row of rows) { const fromFilePath = getFilePath(fromType, fromSubType, fromEntityId, row.filename); const toFilePath = getFilePath(toType, toSubType, toEntityId, row.filename); @@ -339,4 +370,6 @@ module.exports.getFileUrl = getFileUrl; module.exports.getFilePath = getFilePath; module.exports.copyAllTx = copyAllTx; module.exports.removeAllTx = removeAllTx; +module.exports.lockTx = lockTx; +module.exports.unlockTx = unlockTx; module.exports.ReplacementBehavior = ReplacementBehavior; diff --git a/server/models/links.js b/server/models/links.js index 9c6acc0d..3eb585f2 100644 --- a/server/models/links.js +++ b/server/models/links.js @@ -11,6 +11,7 @@ const he = require('he'); const { getPublicUrl } = require('../lib/urls'); const tools = require('../lib/tools'); const shortid = require('shortid'); +const {enforce} = require('../lib/helpers'); const LinkId = { OPEN: -1, @@ -103,16 +104,16 @@ async function countLink(remoteIp, userAgent, campaignCid, listCid, subscription } async function addOrGet(campaignId, url) { - return await knex.transaction(async tx => { - const link = await tx('links').select(['id', 'cid']).where({ - campaign: campaignId, - url - }).first(); + const link = await knex('links').select(['id', 'cid']).where({ + campaign: campaignId, + url + }).first(); - if (!link) { - let cid = shortid.generate(); + if (!link) { + let cid = shortid.generate(); - const ids = await tx('links').insert({ + try { + const ids = await knex('links').insert({ campaign: campaignId, cid, url @@ -122,10 +123,21 @@ async function addOrGet(campaignId, url) { id: ids[0], cid }; - } else { - return link; + } catch (err) { + if (err.code === 'ER_DUP_ENTRY') { + const link = await knex('links').select(['id', 'cid']).where({ + campaign: campaignId, + url + }).first(); + + enforce(link); + return link; + } } - }); + + } else { + return link; + } } async function updateLinks(campaign, list, subscription, mergeTags, message) { diff --git a/server/models/send-configurations.js b/server/models/send-configurations.js index 7e0ca860..7fa4c909 100644 --- a/server/models/send-configurations.js +++ b/server/models/send-configurations.js @@ -14,7 +14,7 @@ const mailers = require('../lib/mailers'); const senders = require('../lib/senders'); const dependencyHelpers = require('../lib/dependency-helpers'); -const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable', 'x_mailer', 'verp_hostname', 'verp_disable_sender_header', 'mailer_type', 'mailer_settings', 'namespace']); +const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'x_mailer', 'verp_hostname', 'verp_disable_sender_header', 'mailer_type', 'mailer_settings', 'namespace']); const allowedMailerTypes = new Set(Object.values(MailerType)); @@ -75,7 +75,7 @@ async function _getByTx(tx, context, key, id, withPermissions, withPrivateData) entity.mailer_settings = JSON.parse(entity.mailer_settings); } else { entity = await tx('send_configurations').where(key, id).select( - ['id', 'name', 'cid', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable'] + ['id', 'name', 'cid', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable'] ).first(); if (!entity) { diff --git a/server/models/subscriptions.js b/server/models/subscriptions.js index f9276073..3c02b0c5 100644 --- a/server/models/subscriptions.js +++ b/server/models/subscriptions.js @@ -19,6 +19,8 @@ const lists = require('./lists'); const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']); +const TEST_USERS_LIST_LIMIT = 1000; + const fieldTypes = {}; const Cardinality = { @@ -409,6 +411,32 @@ async function list(context, listId, grouped, offset, limit) { }); } +async function listTestUsersTx(tx, context, listId, segmentId, grouped) { + await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions'); + + let entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc').where('is_test', true).limit(TEST_USERS_LIST_LIMIT); + + if (segmentId) { + const addSegmentQuery = await segments.getQueryGeneratorTx(tx, listId, segmentId); + + entitiesQry = entitiesQry.where(function() { + addSegmentQuery(this); + }); + } + + const entities = await entitiesQry; + + if (grouped) { + const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId); + + for (const entity of entities) { + groupSubscription(groupedFieldsMap, entity); + } + } + + return entities; +} + // Note that this does not do all the work in the transaction. Thus it is prone to fail if the list is deleted in during the run of the function async function* listIterator(context, listId, segmentId, grouped = true) { let groupedFieldsMap; @@ -855,6 +883,7 @@ module.exports.getByEmail = getByEmail; module.exports.list = list; module.exports.listIterator = listIterator; module.exports.listDTAjax = listDTAjax; +module.exports.listTestUsersTx = listTestUsersTx; module.exports.listTestUsersDTAjax = listTestUsersDTAjax; module.exports.serverValidate = serverValidate; module.exports.create = create; diff --git a/server/models/templates.js b/server/models/templates.js index b18b7186..c04b2cca 100644 --- a/server/models/templates.js +++ b/server/models/templates.js @@ -7,11 +7,17 @@ const dtHelpers = require('../lib/dt-helpers'); const interoperableErrors = require('../../shared/interoperable-errors'); const namespaceHelpers = require('../lib/namespace-helpers'); const shares = require('./shares'); -const reports = require('./reports'); const files = require('./files'); const dependencyHelpers = require('../lib/dependency-helpers'); const {convertFileURLs} = require('../lib/campaign-content'); +const mailers = require('../lib/mailers'); +const tools = require('../lib/tools'); +const sendConfigurations = require('./send-configurations'); +const { getMergeTagsForBases } = require('../../shared/templates'); +const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls'); +const htmlToText = require('html-to-text'); + const allowedKeys = new Set(['name', 'description', 'type', 'data', 'html', 'text', 'namespace']); function hash(entity) { @@ -145,6 +151,65 @@ async function remove(context, id) { }); } +const MAX_EMAIL_COUNT = 100; +async function sendAsTransactionalEmail(context, templateId, sendConfigurationId, emails, subject, mergeTags) { + // TODO - Update this to use CampaignSender.queueMessageTx (with renderedHtml and renderedText) + + if (emails.length > MAX_EMAIL_COUNT) { + throw new Error(`Cannot send more than ${MAX_EMAIL_COUNT} emails at once`); + } + + await knex.transaction(async tx => { + const template = await getByIdTx(tx, context, templateId,false); + const sendConfiguration = await sendConfigurations.getByIdTx(tx, context, sendConfigurationId, false, false); + + await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', sendConfigurationId, 'sendWithoutOverrides'); + + const mailer = await mailers.getOrCreateMailer(sendConfigurationId); + + const variablesSkeleton = { + ...getMergeTagsForBases(getTrustedUrl(), getSandboxUrl(), getPublicUrl()), + ...mergeTags + }; + + for (const email of emails) { + const variables = { + ...variablesSkeleton, + EMAIL: email + }; + + const html = tools.formatTemplate( + template.html, + null, + variables, + true + ); + + const text = (template.text || '').trim() + ? tools.formatTemplate( + template.text, + null, + variables, + false + ) : htmlToText.fromString(html, {wordwrap: 130}); + + return mailer.sendTransactionalMail( + { + to: email, + subject, + from: { + name: sendConfiguration.from_name, + address: sendConfiguration.from_email + }, + html, + text + } + ); + } + }); +} + + module.exports.hash = hash; module.exports.getByIdTx = getByIdTx; module.exports.getById = getById; @@ -153,3 +218,4 @@ module.exports.listByNamespaceDTAjax = listByNamespaceDTAjax; module.exports.create = create; module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck; module.exports.remove = remove; +module.exports.sendAsTransactionalEmail = sendAsTransactionalEmail; diff --git a/server/models/triggers.js b/server/models/triggers.js index 5a02de7b..b8434e24 100644 --- a/server/models/triggers.js +++ b/server/models/triggers.js @@ -65,7 +65,7 @@ async function _validateAndPreprocess(tx, context, campaignId, entity) { await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.source_campaign, 'view'); } - await campaigns.enforceSendPermissionTx(tx, context, campaignId); + await campaigns.enforceSendPermissionTx(tx, context, campaignId, false); } async function create(context, campaignId, entity) { diff --git a/server/models/users.js b/server/models/users.js index 5b7da1bd..038fc045 100644 --- a/server/models/users.js +++ b/server/models/users.js @@ -311,7 +311,7 @@ async function sendPasswordReset(locale, usernameOrEmail) { const { adminEmail } = await settings.get(contextHelpers.getAdminContext(), ['adminEmail']); const mailer = await mailers.getOrCreateMailer(); - await mailer.sendTransactionalMail({ + await mailer.sendTransactionalMailBasedOnTemplate({ to: { address: user.email }, diff --git a/server/routes/api.js b/server/routes/api.js index ebb822e4..0a8f4df4 100644 --- a/server/routes/api.js +++ b/server/routes/api.js @@ -16,8 +16,10 @@ const contextHelpers = require('../lib/context-helpers'); const shares = require('../models/shares'); const slugify = require('slugify'); const passport = require('../lib/passport'); -const TemplateSender = require('../lib/template-sender'); +const templates = require('../models/templates'); const campaigns = require('../models/campaigns'); +const {castToInteger} = require('../lib/helpers'); +const {getSystemSendConfigurationId} = require('../../shared/send-configurations'); class APIError extends Error { constructor(msg, status) { @@ -236,7 +238,7 @@ router.postAsync('/blacklist/add', passport.loggedIn, async (req, res) => { input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim(); }); if (!(input.EMAIL) || (input.EMAIL === '')) { - throw new Error('EMAIL argument is required'); + throw new APIError('EMAIL argument is required', 400); } await blacklist.add(req.context, input.EMAIL); @@ -253,7 +255,7 @@ router.postAsync('/blacklist/delete', passport.loggedIn, async (req, res) => { input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim(); }); if (!(input.EMAIL) || (input.EMAIL === '')) { - throw new Error('EMAIL argument is required'); + throw new APIError('EMAIL argument is required', 400); } await blacklist.remove(req.oontext, input.EMAIL); @@ -288,32 +290,30 @@ router.getAsync('/rss/fetch/:campaignCid', passport.loggedIn, async (req, res) = router.postAsync('/templates/:templateId/send', async (req, res) => { const input = {}; - Object.keys(req.body).forEach(key => { - input[ - (key || '') - .toString() - .trim() - .toUpperCase() - ] = req.body[key] || ''; - }); - try { - const templateSender = new TemplateSender({ - context: req.context, - locale: req.locale, - templateId: req.params.templateId - }); - const info = await templateSender.send({ - data: input.DATA, - email: input.EMAIL, - sendConfigurationId: input.SEND_CONFIGURATION_ID, - subject: input.SUBJECT, - variables: input.VARIABLES - }); - res.status(200).json({ data: info }); - } catch (e) { - throw new APIError(e.message, 400); + for (const key in req.body) { + const sanitizedKey = key.toString().trim().toUpperCase(); + input[sanitizedKey] = req.body[key] || ''; } + + const templateId = castToInteger(req.params.templateId, 'Invalid template ID'); + + let sendConfigurationId; + if (!('SEND_CONFIGURATION_ID' in input)) { + sendConfigurationId = getSystemSendConfigurationId(); + } else { + sendConfigurationId = castToInteger(input.SEND_CONFIGURATION_ID, 'Invalid send configuration ID'); + } + + if (!input.EMAIL || input.EMAIL === 0) { + throw new APIError('Missing email(s)', 400); + } + + const emails = input.EMAIL.split(','); + + const info = await templates.sendAsTransactionalEmail(req.context, templateId, sendConfigurationId, emails, input.SUBJECT, input.VARIABLES); + + res.json({ data: info }); }); module.exports = router; diff --git a/server/routes/archive.js b/server/routes/archive.js index 5e77de78..566fa146 100644 --- a/server/routes/archive.js +++ b/server/routes/archive.js @@ -1,12 +1,12 @@ 'use strict'; const router = require('../lib/router-async').create(); -const CampaignSender = require('../lib/campaign-sender'); +const {CampaignSender} = require('../lib/campaign-sender'); router.get('/:campaign/:list/:subscription', (req, res, next) => { const cs = new CampaignSender(); - cs.init({campaignCid: req.params.campaign}) + cs.initByCampaignCid(req.params.campaign) .then(() => cs.getMessage(req.params.list, req.params.subscription)) .then(result => { const {html} = result; diff --git a/server/routes/rest/campaigns.js b/server/routes/rest/campaigns.js index 77f5a083..f9009e9a 100644 --- a/server/routes/rest/campaigns.js +++ b/server/routes/rest/campaigns.js @@ -114,6 +114,12 @@ router.postAsync('/campaigns-link-clicks-table/:campaignId', passport.loggedIn, return res.json(await campaigns.listLinkClicksDTAjax(req.context, castToInteger(req.params.campaignId), req.body)); }); +router.postAsync('/campaign-test-send', passport.loggedIn, passport.csrfProtection, async (req, res) => { + const data = req.body; + const result = await campaigns.testSend(req.context, data); + return res.json(result); +}); + module.exports = router; \ No newline at end of file diff --git a/server/routes/rest/templates.js b/server/routes/rest/templates.js index 67f666a6..ee7ec252 100644 --- a/server/routes/rest/templates.js +++ b/server/routes/rest/templates.js @@ -5,7 +5,6 @@ const templates = require('../../models/templates'); const router = require('../../lib/router-async').create(); const {castToInteger} = require('../../lib/helpers'); -const CampaignSender = require('../../lib/campaign-sender'); router.getAsync('/templates/:templateId', passport.loggedIn, async (req, res) => { @@ -39,10 +38,4 @@ router.postAsync('/templates-by-namespace-table/:namespaceId', passport.loggedIn return res.json(await templates.listByNamespaceDTAjax(req.context, castToInteger(req.params.namespaceId), req.body)); }); -router.postAsync('/template-test-send', passport.loggedIn, passport.csrfProtection, async (req, res) => { - const data = req.body; - const result = await CampaignSender.testSend(req.context, data.listCid, data.subscriptionCid, data.campaignId, data.sendConfigurationId, data.html, data.text); - return res.json(result); -}); - module.exports = router; \ No newline at end of file diff --git a/server/services/feedcheck.js b/server/services/feedcheck.js index 170b65d7..ed23ee5e 100644 --- a/server/services/feedcheck.js +++ b/server/services/feedcheck.js @@ -112,7 +112,7 @@ async function run() { from_name_override: rssCampaign.from_name_override, from_email_override: rssCampaign.from_email_override, reply_to_override: rssCampaign.reply_to_override, - subject_override: rssCampaign.subject_override, + subject: rssCampaign.subject, data: campaignData, click_tracking_disabled: rssCampaign.click_tracking_disabled, diff --git a/server/services/sender-master.js b/server/services/sender-master.js index 1bec702d..05f4aaf5 100644 --- a/server/services/sender-master.js +++ b/server/services/sender-master.js @@ -6,35 +6,63 @@ const log = require('../lib/log'); const path = require('path'); const knex = require('../lib/knex'); const {CampaignStatus, CampaignType} = require('../../shared/campaigns'); -const { enforce } = require('../lib/helpers'); const campaigns = require('../models/campaigns'); const builtinZoneMta = require('../lib/builtin-zone-mta'); const {CampaignActivityType} = require('../../shared/activity-log'); const activityLog = require('../lib/activity-log'); +const {MessageType} = require('../lib/campaign-sender') require('../lib/fork'); +class Notifications { + constructor() { + this.conts = new Map(); + } + + notify(id) { + const cont = this.conts.get(id); + if (cont) { + for (const cb of cont) { + setImmediate(cb); + } + this.conts.delete(id); + } + } + + async waitFor(id) { + let cont = this.conts.get(id); + if (!cont) { + cont = []; + } + + const notified = new Promise(resolve => { + cont.push(resolve); + }); + + this.conts.set(id, cont); + + await notified; + } +} + +const notifier = new Notifications(); let messageTid = 0; const workerProcesses = new Map(); +const workersCount = config.queue.processes; const idleWorkers = []; let campaignSchedulerRunning = false; let queuedSchedulerRunning = false; -let workerSchedulerRunning = false; -const campaignsCheckPeriod = 5 * 1000; +const campaignsCheckPeriod = 30 * 1000; const retrieveBatchSize = 1000; -const workerBatchSize = 100; +const workerBatchSize = 10; -const messageQueue = new Map(); // campaignId -> [{listId, email}] -const messageQueueCont = new Map(); // campaignId -> next batch callback -const campaignFinishCont = new Map(); // campaignId -> worker finished callback +const sendConfigurationMessageQueue = new Map(); // sendConfigurationId -> [{queuedMessage}] +const campaignMessageQueue = new Map(); // campaignId -> [{listId, email}] -const workAssignment = new Map(); // workerId -> { campaignId, subscribers: [{listId, email}] } - -let workerSchedulerCont = null; -let queuedLastId = 0; +const workAssignment = new Map(); // workerId -> { campaignId, messages: [{listId, email} } / { sendConfigurationId, messages: [{queuedMessage}] } function messagesProcessed(workerId) { @@ -43,108 +71,151 @@ function messagesProcessed(workerId) { workAssignment.delete(workerId); idleWorkers.push(workerId); - if (workerSchedulerCont) { - const cont = workerSchedulerCont; - setImmediate(workerSchedulerCont); - workerSchedulerCont = null; - } - - if (campaignFinishCont.has(wa.campaignId)) { - setImmediate(campaignFinishCont.get(wa.campaignId)); - campaignFinishCont.delete(wa.campaignId); - } + notifier.notify('workerFinished'); } -async function scheduleWorkers() { +async function workersLoop() { + const reservedWorkersForTestCount = workersCount > 1 ? 1 : 0; + async function getAvailableWorker() { - if (idleWorkers.length > 0) { - return idleWorkers.shift(); - - } else { - const workerAvailable = new Promise(resolve => { - workerSchedulerCont = resolve; - }); - - await workerAvailable; - return idleWorkers.shift(); + while (idleWorkers.length === 0) { + await notifier.waitFor('workerFinished'); } + + return idleWorkers.shift(); } + function assignCampaignTaskToWorker(workerId, task) { + const campaignId = task.campaignId; + const queue = task.queue; - if (workerSchedulerRunning) { - return; + const messages = queue.splice(0, workerBatchSize); + workAssignment.set(workerId, {campaignId, messages}); + + if (queue.length === 0) { + notifier.notify(`campaignMessageQueueEmpty:${campaignId}`); + } + + sendToWorker(workerId, 'process-campaign-messages', { + campaignId, + messages + }); } - workerSchedulerRunning = true; - let workerId = await getAvailableWorker(); + function assignSendConfigurationTaskToWorker(workerId, task) { + const sendConfigurationId = task.sendConfigurationId; + const queue = task.queue; - let keepLooping = true; + const messages = queue.splice(0, workerBatchSize); + workAssignment.set(workerId, {sendConfigurationId, messages}); - while (keepLooping) { - keepLooping = false; + if (queue.length === 0) { + notifier.notify(`sendConfigurationMessageQueueEmpty:${sendConfigurationId}`); + } - for (const campaignId of messageQueue.keys()) { - const queue = messageQueue.get(campaignId); + sendToWorker(workerId, 'process-queued-messages', { + sendConfigurationId, + messages + }); + } - if (queue.length > 0) { - const subscribers = queue.splice(0, workerBatchSize); - workAssignment.set(workerId, {campaignId, subscribers}); + function selectNextTask() { + const allocationMap = new Map(); + const allocation = []; - if (queue.length === 0 && messageQueueCont.has(campaignId)) { - setImmediate(messageQueueCont.get(campaignId)); - messageQueueCont.delete(campaignId); + function initAllocation(attrName, queues, assignWorkerHandler) { + for (const id of queues.keys()) { + const key = attrName + ':' + id; + + const queue = queues.get(id); + + const task = { + [attrName]: id, + existingWorkers: 0, + isEmpty: queue.length === 0, + queue, + assignWorkerHandler + }; + + allocationMap.set(key, task); + allocation.push(task); + } + + for (const wa of workAssignment.values()) { + if (wa[attrName]) { + const key = attrName + ':' + wa[attrName]; + const task = allocationMap.get(key); + task.existingWorkers += 1; } - - sendToWorker(workerId, 'process-messages', { - campaignId, - subscribers - }); - workerId = await getAvailableWorker(); - - keepLooping = true; } } + + initAllocation('sendConfigurationId', sendConfigurationMessageQueue, assignSendConfigurationTaskToWorker); + initAllocation('campaignId', campaignMessageQueue, assignCampaignTaskToWorker); + + let minTask = null; + let minExistingWorkers; + + for (const task of allocation) { + if (!task.isEmpty && (minTask === null || minExistingWorkers > task.existingWorkers)) { + minTask = task; + minExistingWorkers = task.existingWorkers; + } + } + + return minTask; } - idleWorkers.push(workerId); - workerSchedulerRunning = false; + while (true) { + const task = selectNextTask(); + + if (task) { + const workerId = await getAvailableWorker(); + task.assignWorkerHandler(workerId, task); + + } else { + await notifier.waitFor('workAvailable'); + } + } } async function processCampaign(campaignId) { - async function finish() { - let workerRunning = false; - for (const wa of workAssignment.values()) { - if (wa.campaignId === campaignId) { - workerRunning = true; + const msgQueue = campaignMessageQueue.get(campaignId); + + async function finish(newStatus) { + const isCompleted = () => { + if (msgQueue.length > 0) return false; + + let workerRunning = false; + + for (const wa of workAssignment.values()) { + if (wa.campaignId === campaignId) { + workerRunning = true; + } } + + return !workerRunning; + }; + + while (!isCompleted()) { + await notifier.waitFor('workerFinished'); } - if (workerRunning) { - const workerFinished = new Promise(resolve => { - campaignFinishCont.set(campaignId, resolve); - }); + campaignMessageQueue.delete(campaignId); - await workerFinished; - setImmediate(finish); - } - - await knex('campaigns').where('id', campaignId).update({status: CampaignStatus.FINISHED}); - await activityLog.logEntityActivity('campaign', CampaignActivityType.STATUS_CHANGE, campaignId, {status: CampaignStatus.FINISHED}); - - messageQueue.delete(campaignId); + await knex('campaigns').where('id', campaignId).update({status: newStatus}); + await activityLog.logEntityActivity('campaign', CampaignActivityType.STATUS_CHANGE, campaignId, {status: newStatus}); } - const msgQueue = []; - messageQueue.set(campaignId, msgQueue); - try { while (true) { const cpg = await knex('campaigns').where('id', campaignId).first(); - if (cpg.status === CampaignStatus.PAUSED) { - messageQueue.delete(campaignId); + if (cpg.status === CampaignStatus.PAUSING) { + msgQueue.splice(0); + await finish(CampaignStatus.PAUSED); return; } @@ -154,21 +225,21 @@ async function processCampaign(campaignId) { }); if (qryGen) { - let subscribersInProcessing = [...msgQueue]; + let messagesInProcessing = [...msgQueue]; for (const wa of workAssignment.values()) { if (wa.campaignId === campaignId) { - subscribersInProcessing = subscribersInProcessing.concat(wa.subscribers); + messagesInProcessing = messagesInProcessing.concat(wa.messages); } } const qry = qryGen(knex) - .whereNotIn('pending_subscriptions.email', subscribersInProcessing.map(x => x.email)) + .whereNotIn('pending_subscriptions.email', messagesInProcessing.map(x => x.email)) .select(['pending_subscriptions.email', 'campaign_lists.list']) .limit(retrieveBatchSize); const subs = await qry; if (subs.length === 0) { - await finish(); + await finish(CampaignStatus.FINISHED); return; } @@ -179,21 +250,19 @@ async function processCampaign(campaignId) { }); } - const nextBatchNeeded = new Promise(resolve => { - messageQueueCont.set(campaignId, resolve); - }); + notifier.notify('workAvailable'); - setImmediate(scheduleWorkers); - - await nextBatchNeeded; + while (msgQueue.length > 0) { + await notifier.waitFor(`campaignMessageQueueEmpty:${campaignId}`); + } } else { - await finish(); + await finish(CampaignStatus.FINISHED); return; } } } catch (err) { - log.error('Senders', `Sending campaign ${campaignId} failed with error: ${err.message}`) + log.error('Senders', `Sending campaign ${campaignId} failed with error: ${err.message}`); log.verbose(err.stack); } } @@ -226,6 +295,8 @@ async function scheduleCampaigns() { }); if (campaignId) { + campaignMessageQueue.set(campaignId, []); + // noinspection JSIgnoredPromiseFromCall processCampaign(campaignId); @@ -234,16 +305,81 @@ async function scheduleCampaigns() { } } } catch (err) { - log.error('Senders', `Scheduling campaigns failed with error: ${err.message}`) + log.error('Senders', `Scheduling campaigns failed with error: ${err.message}`); log.verbose(err.stack); } - campaignSchedulerRunning = false; } -async function processQueued() { +async function processQueuedBySendConfiguration(sendConfigurationId) { + const msgQueue = sendConfigurationMessageQueue.get(sendConfigurationId); + + const isCompleted = () => { + if (msgQueue.length > 0) return false; + + let workerRunning = false; + + for (const wa of workAssignment.values()) { + if (wa.sendConfigurationId === sendConfigurationId) { + workerRunning = true; + } + } + + return !workerRunning; + }; + + try { + while (true) { + let messagesInProcessing = [...msgQueue]; + for (const wa of workAssignment.values()) { + if (wa.sendConfigurationId === sendConfigurationId) { + messagesInProcessing = messagesInProcessing.concat(wa.messages); + } + } + + const rows = await knex('queued') + .orderByRaw(`FIELD(type, ${MessageType.TRIGGERED}, ${MessageType.TEST}) DESC, id ASC`) // This orders MessageType.TEST messages before MessageType.TRIGGERED ones + .where('send_configuration', sendConfigurationId) + .whereNotIn('id', messagesInProcessing.map(x => x.queuedMessage.id)) + .limit(retrieveBatchSize); + + if (rows.length === 0) { + if (isCompleted()) { + sendConfigurationMessageQueue.delete(sendConfigurationId); + return; + + } else { + while (!isCompleted()) { + await notifier.waitFor('workerFinished'); + } + + // At this point, there might be new messages in the queued that could belong to us. Thus we have to try again instead for returning. + continue; + } + } + + for (const row of rows) { + row.data = JSON.parse(row.data); + msgQueue.push({ + queuedMessage: row + }); + } + + notifier.notify('workAvailable'); + + while (msgQueue.length > 0) { + await notifier.waitFor(`sendConfigurationMessageQueueEmpty:${sendConfigurationId}`); + } + } + } catch (err) { + log.error('Senders', `Sending queued messages for send configuration ${sendConfigurationId} failed with error: ${err.message}`); + log.verbose(err.stack); + } +} + +async function scheduleQueued() { if (queuedSchedulerRunning) { return; } @@ -252,34 +388,23 @@ async function processQueued() { try { while (true) { - const rows = await knex('queued') - .orderBy('id', 'asc') - .where('id', '>', queuedLastId) - .limit(retrieveBatchSize); + const sendConfigurationsInProcessing = [...sendConfigurationMessageQueue.keys()]; - if (rows.length === 0) { - break; - } + const rows = await knex('queued') + .whereNotIn('send_configuration', sendConfigurationsInProcessing) + .groupBy('send_configuration') + .select(['send_configuration']); for (const row of rows) { - let msgQueue = messageQueue.get(row.campaign); - if (!msgQueue) { - msgQueue = []; - messageQueue.set(row.campaign, msgQueue); - } + const sendConfigurationId = row.send_configuration; + sendConfigurationMessageQueue.set(sendConfigurationId, []); - msgQueue.push({ - listId: row.list, - subscriptionId: row.subscription - }); + // noinspection JSIgnoredPromiseFromCall + processQueuedBySendConfiguration(sendConfigurationId); } - - queuedLastId = rows[rows.length - 1].id; - - setImmediate(scheduleWorkers); } } catch (err) { - log.error('Senders', `Processing queued messages failed with error: ${err.message}`) + log.error('Senders', `Scheduling queued messages failed with error: ${err.message}`); log.verbose(err.stack); } @@ -337,7 +462,7 @@ function periodicCampaignsCheck() { scheduleCampaigns(); // noinspection JSIgnoredPromiseFromCall - processQueued(); + scheduleQueued(); setTimeout(periodicCampaignsCheck, campaignsCheckPeriod); } @@ -345,7 +470,7 @@ function periodicCampaignsCheck() { async function init() { const spawnWorkerFutures = []; let workerId; - for (workerId = 0; workerId < config.queue.processes; workerId++) { + for (workerId = 0; workerId < workersCount; workerId++) { spawnWorkerFutures.push(spawnWorker(workerId)); } @@ -358,6 +483,7 @@ async function init() { if (type === 'schedule-check') { // noinspection JSIgnoredPromiseFromCall scheduleCampaigns(); + scheduleQueued(); } else if (type === 'reload-config') { for (const workerId of workerProcesses.keys()) { @@ -376,6 +502,8 @@ async function init() { }); periodicCampaignsCheck(); + + setImmediate(workersLoop); } // noinspection JSIgnoredPromiseFromCall diff --git a/server/services/sender-worker.js b/server/services/sender-worker.js index cbe7750f..a9f4c3d8 100644 --- a/server/services/sender-worker.js +++ b/server/services/sender-worker.js @@ -3,14 +3,13 @@ const config = require('config'); const log = require('../lib/log'); const mailers = require('../lib/mailers'); -const CampaignSender = require('../lib/campaign-sender'); -const {enforce} = require('../lib/helpers'); +const {CampaignSender} = require('../lib/campaign-sender'); require('../lib/fork'); const workerId = Number.parseInt(process.argv[2]); let running = false; -async function processMessages(campaignId, subscribers) { +async function processCampaignMessages(campaignId, messages) { if (running) { log.error('Senders', `Worker ${workerId} assigned work while working`); return; @@ -19,23 +18,40 @@ async function processMessages(campaignId, subscribers) { running = true; const cs = new CampaignSender(); - await cs.init({campaignId}) + await cs.initByCampaignId(campaignId); - for (const subData of subscribers) { + for (const msgData of messages) { try { - if (subData.email) { - await cs.sendMessageByEmail(subData.listId, subData.email); + await cs.sendRegularMessage(msgData.listId, msgData.email); - } else if (subData.subscriptionId) { - await cs.sendMessageBySubscriptionId(subData.listId, subData.subscriptionId); - - } else { - enforce(false); - } - - log.verbose('Senders', 'Message sent and status updated for %s:%s', subData.listId, subData.email || subData.subscriptionId); + log.verbose('Senders', 'Message sent and status updated for %s:%s', msgData.listId, msgData.email); } catch (err) { - log.error('Senders', `Sending message to ${subData.listId}:${subData.email} failed with error: ${err.message}`) + log.error('Senders', `Sending message to ${msgData.listId}:${msgData.email} failed with error: ${err.message}`); + log.verbose(err.stack); + } + } + + running = false; + + sendToMaster('messages-processed'); +} + +async function processQueuedMessages(sendConfigurationId, messages) { + if (running) { + log.error('Senders', `Worker ${workerId} assigned work while working`); + return; + } + + running = true; + + for (const msgData of messages) { + const queuedMessage = msgData.queuedMessage; + try { + await CampaignSender.sendQueuedMessage(queuedMessage); + + log.verbose('Senders', 'Message sent and status updated for %s:%s', queuedMessage.list, queuedMessage.subscription); + } catch (err) { + log.error('Senders', `Sending message to ${queuedMessage.list}:${queuedMessage.subscription} failed with error: ${err.message}`); log.verbose(err.stack); } } @@ -58,11 +74,14 @@ process.on('message', msg => { if (type === 'reload-config') { mailers.invalidateMailer(msg.data.sendConfigurationId); - } else if (type === 'process-messages') { + } else if (type === 'process-campaign-messages') { // noinspection JSIgnoredPromiseFromCall - processMessages(msg.data.campaignId, msg.data.subscribers) - } + processCampaignMessages(msg.data.campaignId, msg.data.messages) + } else if (type === 'process-queued-messages') { + // noinspection JSIgnoredPromiseFromCall + processQueuedMessages(msg.data.sendConfigurationId, msg.data.messages) + } } }); diff --git a/server/services/triggers.js b/server/services/triggers.js index a3f11de9..3689f425 100644 --- a/server/services/triggers.js +++ b/server/services/triggers.js @@ -10,6 +10,7 @@ const { Entity, Event } = require('../../shared/triggers'); const { SubscriptionStatus } = require('../../shared/lists'); const links = require('../models/links'); const contextHelpers = require('../lib/context-helpers'); +const {MessageType, CampaignSender} = require('../lib/campaign-sender'); const triggerCheckPeriod = 30 * 1000; const triggerFirePeriod = 120 * 1000; @@ -151,12 +152,13 @@ async function run() { subscription: subscriber.id }); - await tx('queued').insert({ - campaign: campaign.id, - list: cpgList.list, - subscription: subscriber.id, - trigger: trigger.id - }); + await CampaignSender.queueMessageTx(tx, + campaign.send_configuration, cpgList.list, subscriber.id, MessageType.TRIGGERED, + { + campaignId: campaign.id, + triggerId: trigger.id + } + ); await tx('triggers').increment('count').where('id', trigger.id); diff --git a/server/setup/knex/migrations/20170506102634_v1_to_v2.js b/server/setup/knex/migrations/20170506102634_v1_to_v2.js index 05369236..9e1c2b45 100644 --- a/server/setup/knex/migrations/20170506102634_v1_to_v2.js +++ b/server/setup/knex/migrations/20170506102634_v1_to_v2.js @@ -753,11 +753,11 @@ async function migrateSettings(knex) { if (settings.dkimApiKey) { mailer_type = MailerType.ZONE_MTA; - mailer_settings.dkimApiKey = settings.dkimApiKey; + mailer_settings.dkimApiKey = settings.dkimApiKey || ''; mailer_settings.zoneMtaType = ZoneMTAType.WITH_HTTP_CONF; - mailer_settings.dkimDomain = settings.dkimDomain; - mailer_settings.dkimSelector = settings.dkimSelector; - mailer_settings.dkimPrivateKey = settings.dkimPrivateKey; + mailer_settings.dkimDomain = settings.dkimDomain || ''; + mailer_settings.dkimSelector = settings.dkimSelector || ''; + mailer_settings.dkimPrivateKey = settings.dkimPrivateKey || ''; } } @@ -777,7 +777,7 @@ async function migrateSettings(knex) { verp_hostname: settings.verpUse ? settings.verpHostname : null, mailer_type, mailer_settings: JSON.stringify(mailer_settings), - x_mailer: settings.x_mailer, + x_mailer: settings.x_mailer || '', namespace: getGlobalNamespaceId() }); @@ -810,7 +810,7 @@ async function addFiles(knex) { table.string('mimetype'); table.integer('size'); table.timestamp('created').defaultTo(knex.fn.now()); - table.index(['entity', 'originalname']) + table.index(['entity', 'originalname']); }); } } diff --git a/server/setup/knex/migrations/20190615000000_generalization_of_queued_and_file_locking.js b/server/setup/knex/migrations/20190615000000_generalization_of_queued_and_file_locking.js new file mode 100644 index 00000000..99b2c6d1 --- /dev/null +++ b/server/setup/knex/migrations/20190615000000_generalization_of_queued_and_file_locking.js @@ -0,0 +1,63 @@ +const entityTypesWithFiles = { + campaign: { + file: 'files_campaign_file', + attachment: 'files_campaign_attachment', + }, + template: { + file: 'files_template_file' + }, + mosaico_template: { + file: 'files_mosaico_template_file', + block: 'files_mosaico_template_block' + } +}; + +exports.up = (knex, Promise) => (async() => { + await knex.schema.table('queued', table => { + table.integer('send_configuration').unsigned().notNullable(); + table.integer('type').unsigned().notNullable(); // The values come from campaign-sender.js:MessageType + table.text('data', 'longtext'); + }); + + const queued = await knex('queued') + .leftJoin('campaigns', 'queued.campaign', 'campaigns.id') + .select(['queued.id', 'queued.trigger', 'queued.campaign', 'campaigns.send_configuration']); + + for (const queuedEntry of queued) { + const data = {}; + + if (queued.trigger) { + data.triggerId = queuedEntry.trigger; + data.campaignId = queuedEntry.campaign; + } + + knex('queued') + .where('id', queuedEntry.id) + .update({ + send_configuration: queuedEntry.send_configuration, + data: JSON.stringify(data) + }); + } + + await knex.schema.table('queued', table => { + table.dropColumn('trigger'); + table.dropColumn('campaign'); + }); + + + for (const type in entityTypesWithFiles) { + const typeEntry = entityTypesWithFiles[type]; + + for (const subType in typeEntry) { + const subTypeEntry = typeEntry[subType]; + + await knex.schema.table(subTypeEntry, table => { + table.boolean('delete_pending').notNullable().defaultTo(false); + table.integer('lock_count').notNullable().defaultTo(0); + }); + } + } +})(); + +exports.down = (knex, Promise) => (async() => { +})(); diff --git a/server/setup/knex/migrations/20190616000000_drop_subject_in_send_configurations.js b/server/setup/knex/migrations/20190616000000_drop_subject_in_send_configurations.js new file mode 100644 index 00000000..862a8bad --- /dev/null +++ b/server/setup/knex/migrations/20190616000000_drop_subject_in_send_configurations.js @@ -0,0 +1,15 @@ +exports.up = (knex, Promise) => (async() => { + await knex.schema.table('send_configurations', table => { + table.dropColumn('subject'); + table.dropColumn('subject_overridable'); + }); + + await knex.schema.table('campaigns', table => { + table.renameColumn('subject_override', 'subject'); + }); + + await knex('campaigns').whereNull('subject').update('subject', ''); +})(); + +exports.down = (knex, Promise) => (async() => { +})(); diff --git a/shared/activity-log.js b/shared/activity-log.js index ea029f97..55ea6c63 100644 --- a/shared/activity-log.js +++ b/shared/activity-log.js @@ -8,7 +8,8 @@ const EntityActivityType = { }; const CampaignActivityType = { - STATUS_CHANGE: EntityActivityType.MAX + 1 + STATUS_CHANGE: EntityActivityType.MAX + 1, + TEST_SEND: EntityActivityType.MAX + 2, }; const ListActivityType = { diff --git a/shared/campaigns.js b/shared/campaigns.js index 43d7f28b..ccdbd4b7 100644 --- a/shared/campaigns.js +++ b/shared/campaigns.js @@ -38,11 +38,12 @@ const CampaignStatus = { // For campaign types: NORMAL, RSS_ENTRY SENDING: 7, + PAUSING: 8, - MAX: 8 + MAX: 9 }; -const campaignOverridables = ['from_name', 'from_email', 'reply_to', 'subject']; +const campaignOverridables = ['from_name', 'from_email', 'reply_to']; function getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration) { let allowedOverride = false;