public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Work in progress on refactoring all mail sending to use the message sender an sender workers. No yet finished.

Browse source
This commit is contained in:
Tomas Bures 2019-06-29 23:19:56 +02:00
parent 355e03900a
commit 4e9f6bd57b
22 changed files with 811 additions and 444 deletions
Download patch file Download diff file Expand all files Collapse all files

92
server/lib/builtin-zone-mta.js
View file

@ -4,17 +4,31 @@ const config = require('config');
const fork = require('./fork').fork;
const log = require('./log');
const path = require('path');
const fs = require('fs-extra')
const fs = require('fs-extra');
const crypto = require('crypto');
const bluebird = require('bluebird');
let zoneMtaProcess;
let zoneMtaProcess = null;
const zoneMtaDir = path.join(__dirname, '..', '..', 'zone-mta');
const zoneMtaBuiltingConfig = path.join(zoneMtaDir, 'config', 'builtin-zonemta.json');
const password = process.env.BUILTIN_ZONE_MTA_PASSWORD || crypto.randomBytes(20).toString('hex').toLowerCase();
let restartCount = 0;
let lastRestartCount = 0;
let restartBackoffIdx = 0;
const restartBackoff = [0, 30, 60, 300]; // in seconds
setInterval(() => {
if (restartCount === lastRestartCount) {
restartBackoffIdx = 0;
}
lastRestartCount = restartCount;
}, 300000 /* 5 mins */);
function getUsername() {
return 'mailtrain';
}
@ -119,36 +133,58 @@ async function createConfig() {
await fs.writeFile(zoneMtaBuiltingConfig, JSON.stringify(cnf, null, 2));
}
function restart(callback) {
if (zoneMtaProcess) return callback();
if (restartCount === 0) {
log.info('ZoneMTA', 'Starting built-in Zone MTA process');
} else {
log.info('ZoneMTA', `Restarting built-in Zone MTA process (restart count ${restartCount})`);
}
zoneMtaProcess = fork(
path.join(zoneMtaDir, 'index.js'),
['--config=' + zoneMtaBuiltingConfig],
{
cwd: zoneMtaDir,
env: {NODE_ENV: process.env.NODE_ENV}
}
);
zoneMtaProcess.on('message', msg => {
if (msg) {
if (msg.type === 'zone-mta-started') {
log.info('ZoneMTA', 'ZoneMTA process started');
if (callback) {
return callback();
} else {
return;
}
}
}
});
zoneMtaProcess.on('close', (code, signal) => {
log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal);
zoneMtaProcess = null;
restartCount += 1;
const backoffTimeout = restartBackoff[restartBackoffIdx] * 1000;
if (restartBackoffIdx < restartBackoff.length - 1) {
restartBackoffIdx += 1;
}
setTimeout(restart, backoffTimeout);
});
}
function spawn(callback) {
if (config.builtinZoneMTA.enabled) {
createConfig().then(() => {
log.info('ZoneMTA', 'Starting built-in Zone MTA process');
zoneMtaProcess = fork(
path.join(zoneMtaDir, 'index.js'),
['--config=' + zoneMtaBuiltingConfig],
{
cwd: zoneMtaDir,
env: {NODE_ENV: process.env.NODE_ENV}
}
);
zoneMtaProcess.on('message', msg => {
if (msg) {
if (msg.type === 'zone-mta-started') {
log.info('ZoneMTA', 'ZoneMTA process started');
return callback();
} else if (msg.type === 'entries-added') {
senders.scheduleCheck();
}
}
});
zoneMtaProcess.on('close', (code, signal) => {
log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal);
});
restart(callback);
}).catch(err => callback(err));
} else {

6
server/lib/knex.js
View file

@ -23,6 +23,12 @@ const knex = require('knex')({
//, debug: true
});
/*
This is to enable logging on mysql side:
SET GLOBAL general_log = 'ON';
SET GLOBAL general_log_file = '/tmp/mysql-all.log';
*/
module.exports = knex;

81
server/lib/mailers.js
View file

@ -3,8 +3,6 @@
const log = require('./log');
const config = require('config');
const Handlebars = require('handlebars');
const util = require('util');
const nodemailer = require('nodemailer');
const aws = require('aws-sdk');
const openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
@ -14,13 +12,21 @@ const builtinZoneMta = require('./builtin-zone-mta');
const contextHelpers = require('./context-helpers');
const settings = require('../models/settings');
const tools = require('./tools');
const htmlToText = require('html-to-text');
const bluebird = require('bluebird');
const transports = new Map();
class SendConfigurationError extends Error {
constructor(sendConfigurationId, ...args) {
super(...args);
this.sendConfigurationId = sendConfigurationId;
Error.captureStackTrace(this, SendConfigurationError);
}
}
async function getOrCreateMailer(sendConfigurationId) {
let sendConfiguration;
@ -73,25 +79,18 @@ function _addDkimKeys(transport, mail) {
async function _sendMail(transport, mail, template) {
_addDkimKeys(transport, mail);
let tryCount = 0;
const trySend = (callback) => {
tryCount++;
transport.sendMail(mail, (err, info) => {
if (err) {
log.error('Mail', err);
if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) {
// temporary error, try again
log.verbose('Mail', 'Retrying after %s sec. ...', tryCount);
return setTimeout(trySend, tryCount * 1000);
}
return callback(err);
}
return callback(null, info);
});
};
try {
return await transport.sendMailAsync(mail);
const trySendAsync = bluebird.promisify(trySend);
return await trySendAsync();
} catch (err) {
if ( (err.responseCode && err.responseCode >= 400 && err.responseCode < 500) ||
(err.code === 'ECONNECTION' && err.errno === 'ECONNREFUSED')
) {
throw new SendConfigurationError(transport.mailer.sendConfiguration.id, 'Cannot connect to service specified by send configuration ' + transport.mailer.sendConfiguration.id);
}
throw err;
}
}
async function _sendTransactionalMail(transport, mail) {
@ -103,39 +102,6 @@ async function _sendTransactionalMail(transport, mail) {
return await _sendMail(transport, mail);
}
async function _sendTransactionalMailBasedOnTemplate(transport, mail, template) {
const sendConfiguration = transport.mailer.sendConfiguration;
mail.from = {
name: sendConfiguration.from_name,
address: sendConfiguration.from_email
};
const htmlRenderer = await tools.getTemplate(template.html, template.locale);
if (htmlRenderer) {
mail.html = htmlRenderer(template.data || {});
}
const preparedHtml = await tools.prepareHtml(mail.html);
if (preparedHtml) {
mail.html = preparedHtml;
}
const textRenderer = await tools.getTemplate(template.text, template.locale);
if (textRenderer) {
mail.text = textRenderer(template.data || {});
} else if (mail.html) {
mail.text = htmlToText.fromString(mail.html, {
wordwrap: 130
});
}
return await _sendTransactionalMail(transport, mail);
}
async function _createTransport(sendConfiguration) {
const mailerSettings = sendConfiguration.mailer_settings;
const mailerType = sendConfiguration.mailer_type;
@ -222,6 +188,7 @@ async function _createTransport(sendConfiguration) {
}
const transport = nodemailer.createTransport(transportOptions, config.nodemailer);
transport.sendMailAsync = bluebird.promisify(transport.sendMail.bind(transport));
transport.use('stream', openpgpEncrypt({
signingKey: configItems.pgpPrivateKey,
@ -267,8 +234,7 @@ async function _createTransport(sendConfiguration) {
transport.mailer = {
sendConfiguration,
throttleWait: bluebird.promisify(throttleWait),
sendTransactionalMail: async (mail, template) => await _sendTransactionalMail(transport, mail),
sendTransactionalMailBasedOnTemplate: async (mail, template) => await _sendTransactionalMailBasedOnTemplate(transport, mail, template),
sendTransactionalMail: async (mail) => await _sendTransactionalMail(transport, mail),
sendMassMail: async (mail, template) => await _sendMail(transport, mail)
};
@ -286,3 +252,4 @@ class MailerError extends Error {
module.exports.getOrCreateMailer = getOrCreateMailer;
module.exports.invalidateMailer = invalidateMailer;
module.exports.MailerError = MailerError;
module.exports.SendConfigurationError = SendConfigurationError;

432
server/lib/campaign-sender.js → server/lib/message-sender.js
View file

@ -15,9 +15,9 @@ const links = require('../models/links');
const {CampaignSource, CampaignType} = require('../../shared/campaigns');
const {SubscriptionStatus} = require('../../shared/lists');
const tools = require('./tools');
const htmlToText = require('html-to-text');
const request = require('request-promise');
const files = require('../models/files');
const htmlToText = require('html-to-text');
const {getPublicUrl} = require('./urls');
const blacklist = require('../models/blacklist');
const libmime = require('libmime');
@ -26,10 +26,11 @@ const { enforce } = require('./helpers');
const MessageType = {
REGULAR: 0,
TRIGGERED: 1,
TEST: 2
TEST: 2,
SUBSCRIPTION: 3
};
class CampaignSender {
class MessageSender {
constructor() {
}
@ -40,6 +41,8 @@ class CampaignSender {
- sendConfiguration, listId, attachments, html, text, subject
*/
async _init(settings) {
this.type = settings.type;
this.listsById = new Map(); // listId -> list
this.listsByCid = new Map(); // listCid -> list
this.listsFieldsGrouped = new Map(); // listId -> fieldsGrouped
@ -47,11 +50,13 @@ class CampaignSender {
await knex.transaction(async tx => {
if (settings.campaignCid) {
this.campaign = await campaigns.rawGetByTx(tx, 'cid', settings.campaignCid);
this.isMassMail = true;
} else if (settings.campaignId) {
this.campaign = await campaigns.rawGetByTx(tx, 'id', settings.campaignId);
this.isMassMail = true;
} else {
} else if (this.type === MessageType.TEST) {
// We are not within scope of a campaign (i.e. templates in MessageType.TEST message)
// This is to fake the campaign for getMessageLinks, which is called inside formatMessage
this.campaign = {
@ -60,11 +65,15 @@ class CampaignSender {
from_email_override: null,
reply_to_override: null
};
this.isMassMail = true;
} else {
this.isMassMail = false;
}
if (settings.sendConfigurationId) {
this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), settings.sendConfigurationId, false, true);
} else if (this.campaign.send_configuration) {
} else if (this.campaign && this.campaign.send_configuration) {
this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.send_configuration, false, true);
} else {
enforce(false);
@ -79,7 +88,7 @@ class CampaignSender {
this.listsByCid.set(list.cid, list);
this.listsFieldsGrouped.set(list.id, await fields.listGroupedTx(tx, list.id));
} else if (this.campaign.lists) {
} else if (this.campaign && this.campaign.lists) {
for (const listSpec of this.campaign.lists) {
const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list);
this.listsById.set(list.id, list);
@ -94,7 +103,7 @@ class CampaignSender {
if (settings.attachments) {
this.attachments = settings.attachments;
} else if (this.campaign.id) {
} else if (this.campaign && this.campaign.id) {
const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', this.campaign.id);
this.attachments = [];
@ -109,16 +118,21 @@ class CampaignSender {
this.attachments = [];
}
if (settings.html !== undefined) {
if (settings.renderedHtml !== undefined) {
this.rendereHtml = settings.rendereHtml;
this.renderedText = settings.renderedText;
} else if (settings.html !== undefined) {
this.html = settings.html;
this.text = settings.text;
} else if (this.campaign.source === CampaignSource.TEMPLATE) {
} else if (this.campaign && this.campaign.source === CampaignSource.TEMPLATE) {
this.template = await templates.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.data.sourceTemplate, false);
}
if (settings.subject !== undefined) {
this.subject = settings.subject;
} else if (this.campaign.subject !== undefined) {
} else if (this.campaign && this.campaign.subject !== undefined) {
this.subject = this.campaign.subject;
} else {
enforce(false);
@ -142,7 +156,7 @@ class CampaignSender {
text = this.text;
renderTags = true;
} else {
} else if (campaign) {
if (campaign.source === CampaignSource.URL) {
const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
for (const key in mergeTags) {
@ -222,14 +236,16 @@ class CampaignSender {
}
async initByCampaignCid(campaignCid) {
await this._init({campaignCid});
await this._init({type: MessageType.REGULAR, campaignCid});
}
async initByCampaignId(campaignId) {
await this._init({campaignId});
await this._init({type: MessageType.REGULAR, campaignId});
}
async getMessage(listCid, subscriptionCid) {
enforce(this.type === MessageType.REGULAR);
const list = this.listsByCid.get(listCid);
const subscriptionGrouped = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid);
const flds = this.listsFieldsGrouped.get(list.id);
@ -242,93 +258,81 @@ class CampaignSender {
/*
subData is one of:
- queuedMessage
- subscriptionId, listId, attachments
or
- email, listId
or
- to, subject
*/
async _sendMessage(subData) {
let msgType;
let subscriptionGrouped;
let listId;
let msgType = this.type;
let to, email;
let envelope = false;
let sender = false;
let headers = {};
let listHeader = false;
let encryptionKeys = [];
let subject;
let message;
if (subData.queuedMessage) {
const queuedMessage = subData.queuedMessage;
msgType = queuedMessage.type;
listId = queuedMessage.list;
subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, queuedMessage.subscription);
let subscriptionGrouped, list; // May be undefined
const campaign = this.campaign; // May be undefined
} else {
enforce(subData.email);
enforce(subData.listId);
if (subData.listId) {
let listId;
subscriptionGrouped;
msgType = MessageType.REGULAR;
listId = subData.listId;
subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, subData.email);
}
if (subData.subscriptionId) {
listId = subData.listId;
subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, subData.subscriptionId);
const email = subscriptionGrouped.email;
if (await blacklist.isBlacklisted(email)) {
return;
}
const list = this.listsById.get(listId);
const flds = this.listsFieldsGrouped.get(list.id);
const campaign = this.campaign;
const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign));
const encryptionKeys = [];
for (const fld of flds) {
if (fld.type === 'gpg' && mergeTags[fld.key]) {
encryptionKeys.push(mergeTags[fld.key].trim());
} else if (subData.email) {
listId = subData.listId;
subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, subData.email);
}
}
const sendConfiguration = this.sendConfiguration;
list = this.listsById.get(listId);
email = subscriptionGrouped.email;
const {html, text, attachments} = await this._getMessage(list, subscriptionGrouped, mergeTags, true);
const flds = this.listsFieldsGrouped.get(list.id);
const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign));
const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
let listUnsubscribe = null;
if (!list.listunsubscribe_disabled) {
listUnsubscribe = campaign.unsubscribe_url
? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, campaign.unsubscribe_url)
: getPublicUrl('/subscription/' + list.cid + '/unsubscribe/' + subscriptionGrouped.cid);
}
const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
await mailer.throttleWait();
const getOverridable = key => {
if (sendConfiguration[key + '_overridable'] && this.campaign[key + '_override'] !== null) {
return campaign[key + '_override'] || '';
} else {
return sendConfiguration[key] || '';
for (const fld of flds) {
if (fld.type === 'gpg' && mergeTags[fld.key]) {
encryptionKeys.push(mergeTags[fld.key].trim());
}
}
};
const mail = {
from: {
name: getOverridable('from_name'),
address: getOverridable('from_email')
},
replyTo: getOverridable('reply_to'),
xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
to: {
message = await this._getMessage(list, subscriptionGrouped, mergeTags, true);
const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
let listUnsubscribe = null;
if (!list.listunsubscribe_disabled) {
listUnsubscribe = campaign.unsubscribe_url
? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, campaign.unsubscribe_url)
: getPublicUrl('/subscription/' + list.cid + '/unsubscribe/' + subscriptionGrouped.cid);
}
to = {
name: list.to_name === null ? undefined : tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false),
address: subscriptionGrouped.email
},
sender: this.useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false,
};
envelope: this.useVerp ? {
from: campaignAddress + '@' + sendConfiguration.verp_hostname,
to: subscriptionGrouped.email
} : false,
subject = tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, this.subject, false);
headers: {
if (this.useVerp) {
envelope = {
from: campaignAddress + '@' + sendConfiguration.verp_hostname,
to: subscriptionGrouped.email
};
}
if (this.useVerpSenderHeader) {
sender = campaignAddress + '@' + sendConfiguration.verp_hostname;
}
headers = {
'x-fbl': campaignAddress,
// custom header for SparkPost
'x-msys-api': JSON.stringify({
@ -348,175 +352,237 @@ class CampaignSender {
prepared: true,
value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>'
}
},
list: {
unsubscribe: listUnsubscribe
},
subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, this.subject, false),
html,
text,
};
attachments,
listHeader = {
unsubscribe: listUnsubscribe
};
} else if (subData.to) {
to = subData.to;
email = to.address;
subject = this.subject;
encryptionKeys = subData.encryptionKeys;
message = await this._getMessage();
}
if (await blacklist.isBlacklisted(email)) {
return;
}
const sendConfiguration = this.sendConfiguration;
const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
await mailer.throttleWait();
const getOverridable = key => {
if (campaign && sendConfiguration[key + '_overridable'] && this.campaign[key + '_override'] !== null) {
return campaign[key + '_override'] || '';
} else {
return sendConfiguration[key] || '';
}
};
const mail = {
from: {
name: getOverridable('from_name'),
address: getOverridable('from_email')
},
replyTo: getOverridable('reply_to'),
xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
to,
sender,
envelope,
headers,
list: listHeader,
subject,
html: message.html,
text: message.text,
attachments: message.attachments || [],
encryptionKeys
};
let status;
let response;
let responseId = null;
try {
const info = await mailer.sendMassMail(mail);
status = SubscriptionStatus.SUBSCRIBED;
log.verbose('CampaignSender', `response: ${info.response} messageId: ${info.messageId}`);
const info = this.isMassMail ? await mailer.sendMassMail(mail) : await mailer.sendTransactionalMail(mail);
let match;
if ((match = info.response.match(/^250 Message queued as ([0-9a-f]+)$/))) {
/*
ZoneMTA
info.response: 250 Message queued as 1691ad7f7ae00080fd
info.messageId: <e65c9386-e899-7d01-b21e-ec03c3a9d9b4@sathyasai.org>
*/
response = info.response;
responseId = match[1];
} else if ((match = info.messageId.match(/^<([^>@]*)@.*amazonses\.com>$/))) {
/*
AWS SES
info.response: 0102016ad2244c0a-955492f2-9194-4cd1-bef9-70a45906a5a7-000000
info.messageId: <0102016ad2244c0a-955492f2-9194-4cd1-bef9-70a45906a5a7-000000@eu-west-1.amazonses.com>
*/
response = info.response;
responseId = match[1];
} else if (info.response.match(/^250 OK$/) && (match = info.messageId.match(/^<([^>]*)>$/))) {
/*
Postal Mail Server
info.response: 250 OK
info.messageId: <xxxxxxxxx@xxx.xx> (postal messageId)
*/
response = info.response;
responseId = match[1];
} else {
/*
Fallback - Mailtrain v1 behavior
*/
response = info.response || info.messageId;
responseId = response.split(/\s+/).pop();
}
if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) {
await knex('campaigns').where('id', campaign.id).increment('delivered');
}
} catch (err) {
console.log(err);
log.verbose('MessageSender', `response: ${info.response} messageId: ${info.messageId}`);
let match;
if ((match = info.response.match(/^250 Message queued as ([0-9a-f]+)$/))) {
/*
{ Error: connect ECONNREFUSED 127.0.0.1:55871
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)
cause:
{ Error: connect ECONNREFUSED 127.0.0.1:55871
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)
stack:
'Error: connect ECONNREFUSED 127.0.0.1:55871\n at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1097:14)',
errno: 'ECONNREFUSED',
code: 'ECONNECTION',
syscall: 'connect',
address: '127.0.0.1',
port: 55871,
command: 'CONN' },
isOperational: true,
errno: 'ECONNREFUSED',
code: 'ECONNECTION',
syscall: 'connect',
address: '127.0.0.1',
port: 55871,
command: 'CONN' }
ZoneMTA
info.response: 250 Message queued as 1691ad7f7ae00080fd
info.messageId: <e65c9386-e899-7d01-b21e-ec03c3a9d9b4@sathyasai.org>
*/
response = info.response;
responseId = match[1];
status = SubscriptionStatus.BOUNCED;
response = err.response || err.message;
if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) {
await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced');
}
} else if ((match = info.messageId.match(/^<([^>@]*)@.*amazonses\.com>$/))) {
/*
AWS SES
info.response: 0102016ad2244c0a-955492f2-9194-4cd1-bef9-70a45906a5a7-000000
info.messageId: <0102016ad2244c0a-955492f2-9194-4cd1-bef9-70a45906a5a7-000000@eu-west-1.amazonses.com>
*/
response = info.response;
responseId = match[1];
} else if (info.response.match(/^250 OK$/) && (match = info.messageId.match(/^<([^>]*)>$/))) {
/*
Postal Mail Server
info.response: 250 OK
info.messageId: <xxxxxxxxx@xxx.xx> (postal messageId)
*/
response = info.response;
responseId = match[1];
} else {
/*
Fallback - Mailtrain v1 behavior
*/
response = info.response || info.messageId;
responseId = response.split(/\s+/).pop();
}
if (msgType === MessageType.REGULAR || msgType === MessageType.TRIGGERED) {
await knex('campaigns').where('id', campaign.id).increment('delivered');
}
const now = new Date();
if (msgType === MessageType.REGULAR) {
enforce(list);
enforce(subscriptionGrouped);
await knex('campaign_messages').insert({
campaign: this.campaign.id,
list: list.id,
subscription: subscriptionGrouped.id,
send_configuration: sendConfiguration.id,
status,
status: SubscriptionStatus.SUBSCRIBED,
response,
response_id: responseId,
updated: now
});
} else if (msgType === MessageType.TRIGGERED || msgType === MessageType.TEST) {
if (subData.queuedMessage.data.attachments) {
for (const attachment of subData.queuedMessage.data.attachments) {
} else if (msgType === MessageType.TRIGGERED || msgType === MessageType.TEST || msgType === MessageType.SUBSCRIPTION) {
if (subData.attachments) {
for (const attachment of subData.attachments) {
try {
// We ignore any errors here because we already sent the message. Thus we have to mark it as completed to avoid sending it again.
await knex.transaction(async tx => {
await files.unlockTx(tx, 'campaign', 'attachment', attachment.id);
});
} catch (err) {
log.error('CampaignSender', `Error when unlocking attachment ${attachment.id} for ${listId}:${subscriptionGrouped.email} (queuedId: ${subData.queuedMessage.id})`);
log.error('MessageSender', `Error when unlocking attachment ${attachment.id} for ${email} (queuedId: ${subData.queuedId})`);
log.verbose(err.stack);
}
}
}
await knex('queued')
.where({id: subData.queuedMessage.id})
.where({id: subData.queuedId})
.del();
}
}
async sendRegularMessage(listId, email) {
enforce(this.type === MessageType.REGULAR);
await this._sendMessage({listId, email});
}
}
CampaignSender.sendQueuedMessage = async (queuedMessage) => {
async function sendQueuedMessage(queuedMessage) {
const msgData = queuedMessage.data;
const cs = new CampaignSender();
const cs = new MessageSender();
await cs._init({
type: queuedMessage.type,
campaignId: msgData.campaignId,
listId: queuedMessage.list,
listId: msgData.listId,
sendConfigurationId: queuedMessage.send_configuration,
attachments: msgData.attachments,
html: msgData.html,
text: msgData.text,
subject: msgData.subject
subject: msgData.subject,
renderedHtml: msgData.renderedHtml,
renderedText: msgData.renderedText
});
await cs._sendMessage({queuedMessage});
};
await cs._sendMessage({
subscriptionId: msgData.subscriptionId,
listId: msgData.listId,
to: msgData.to,
attachments: msgData.attachments,
encryptionKeys: msgData.encryptionKeys,
queuedId: queuedMessage.id
});
}
CampaignSender.queueMessageTx = async (tx, sendConfigurationId, listId, subscriptionId, messageType, messageData) => {
if (messageData.attachments) {
async function queueCampaignMessageTx(tx, sendConfigurationId, listId, subscriptionId, messageType, messageData) {
enforce(messageType === MessageType.TRIGGERED || messageType === MessageType.TEST);
const msgData = {...messageData};
if (msgData.attachments) {
for (const attachment of messageData.attachments) {
await files.lockTx(tx,'campaign', 'attachment', attachment.id);
}
}
msgData.listId = listId;
msgData.subscriptionId = subscriptionId;
await tx('queued').insert({
send_configuration: sendConfigurationId,
list: listId,
subscription: subscriptionId,
type: messageType,
data: JSON.stringify(messageData)
data: JSON.stringify(msgData)
});
};
}
module.exports.CampaignSender = CampaignSender;
module.exports.MessageType = MessageType;
async function queueSubscriptionMessage(sendConfigurationId, to, subject, encryptionKeys, template) {
let html, text;
const htmlRenderer = await tools.getTemplate(template.html, template.locale);
if (htmlRenderer) {
html = htmlRenderer(template.data || {});
if (html) {
html = await tools.prepareHtml(html);
}
}
const textRenderer = await tools.getTemplate(template.text, template.locale);
if (textRenderer) {
text = textRenderer(template.data || {});
} else if (html) {
text = htmlToText.fromString(html, {
wordwrap: 130
});
}
const msgData = {
renderedHtml: html,
renderedText: text,
to,
subject,
encryptionKeys
};
await tx('queued').insert({
send_configuration: sendConfigurationId,
type: MessageType.SUBSCRIPTION,
data: JSON.stringify(msgData)
});
}
module.exports.MessageSender = MessageSender;
module.exports.MessageType = MessageType;
module.exports.sendQueuedMessage = sendQueuedMessage;
module.exports.queueCampaignMessageTx = queueCampaignMessageTx;
module.exports.queueSubscriptionMessage = queueSubscriptionMessage;

1
server/lib/senders.js
View file

@ -15,6 +15,7 @@ function spawn(callback) {
log.verbose('Senders', 'Spawning master sender process');
knex('campaigns').where('status', CampaignStatus.SENDING).update({status: CampaignStatus.SCHEDULED})
.then(() => knex('campaigns').where('status', CampaignStatus.PAUSING).update({status: CampaignStatus.PAUSED}))
.then(() => {
senderProcess = fork(path.join(__dirname, '..', 'services', 'sender-master.js'), [], {
cwd: path.join(__dirname, '..'),

26
server/lib/subscription-mail-helpers.js
View file

@ -5,11 +5,10 @@ const fields = require('../models/fields');
const settings = require('../models/settings');
const {getTrustedUrl, getPublicUrl} = require('./urls');
const { tUI, tMark } = require('./translate');
const util = require('util');
const contextHelpers = require('./context-helpers');
const {getFieldColumn} = require('../../shared/lists');
const forms = require('../models/forms');
const mailers = require('./mailers');
const messageSender = require('./message-sender');
module.exports = {
sendAlreadySubscribed,
@ -138,20 +137,21 @@ async function _sendMail(list, email, template, locale, subjectKey, relativeUrls
try {
if (list.send_configuration) {
const mailer = await mailers.getOrCreateMailer(list.send_configuration);
await mailer.sendTransactionalMailBasedOnTemplate({
to: {
await messageSender.queueSubscriptionMessage(
list.send_configuration,
{
name: getDisplayName(flds, subscription),
address: email
},
subject: tUI(subjectKey, locale, { list: list.name }),
encryptionKeys
}, {
html,
text,
locale,
data
});
tUI(subjectKey, locale, { list: list.name }),
encryptionKeys,
{
html,
text,
locale,
data
}
);
} else {
log.warn('Subscription', `Not sending email for list id:${list.id} because not send configuration is set.`);
}

10
server/lib/tools.js
View file

@ -42,9 +42,7 @@ async function getLocalizedFile(basePath, fileName, language) {
}
async function getTemplate(template, locale) {
if (!template) {
return false;
}
enforce(template);
const key = getLangCodeFromExpressLocale(locale) + ':' + ((typeof template === 'object') ? hasher.hash(template) : template);
@ -148,7 +146,7 @@ function validateEmailGetMessage(result, address, language) {
}
function formatMessage(campaign, list, subscription, mergeTags, message, isHTML) {
const links = getMessageLinks(campaign, list, subscription);
const links = campaign && list && subscription ? getMessageLinks(campaign, list, subscription) : {};
return formatTemplate(message, links, mergeTags, isHTML);
}
@ -192,10 +190,6 @@ function formatTemplate(template, links, mergeTags, isHTML) {
}
async function prepareHtml(html) {
if (!(html || '').toString().trim()) {
return false;
}
const { window } = new JSDOM(html);
const head = window.document.querySelector('head');