public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fetch multiple unsent messages at once to speed up delivery

Browse source
This commit is contained in:
Andris Reinman 2016-05-25 18:01:39 +03:00
parent 9a5d723663
commit f29a8a1b67
9 changed files with 184 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
README.md
View file

@ -98,6 +98,18 @@ Mailtrain uses webhooks integration to detect bounces and spam complaints. Curre
Additionally Mailtrain (v1.1+) is able to use VERP-based bounce handling. This would require to have a compatible SMTP relay (the services mentioned above strip out or block VERP addresses in the SMTP envelope) and you also need to set up special MX DNS name that points to your Mailtrain installation server.
## Testing
There is a built in /dev/null server in Mailtrain that you can use to load test your installation. Check the `[testserver]` section in the configuration file for details. By default the test server is disabled. The server uses only cleartext connections, so select "Do not use encryption" in the encryption settings when setting up the server data in Mailtrain.
Additionally you can generate CSV import files with fake subscriber data:
```
node setup/fakedata.js > somefile.csv
```
This command generates a CSV file with 100 000 subscriber accounts
## License
**GPL-V3.0**

3
config/default.toml
View file

@ -51,3 +51,6 @@ host="0.0.0.0"
enabled=false
port=5587
host="0.0.0.0"
username="testuser"
password="testpass"
logger=false

32
lib/models/segments.js
View file

@ -512,7 +512,7 @@ module.exports.deleteRule = (id, callback) => {
});
};
module.exports.getQuery = (id, callback) => {
module.exports.getQuery = (id, prefix, callback) => {
module.exports.get(id, (err, segment) => {
if (err) {
return callback(err);
@ -522,6 +522,8 @@ module.exports.getQuery = (id, callback) => {
return callback(new Error('Segment not found'));
}
prefix = prefix ? prefix + '.' : '';
let query = [];
let values = [];
@ -539,25 +541,25 @@ module.exports.getQuery = (id, callback) => {
segment.rules.forEach(rule => {
switch (rule.columnType.type) {
case 'string':
query.push('`' + rule.columnType.column + '` LIKE ?');
query.push(prefix + '`' + rule.columnType.column + '` LIKE ?');
values.push(rule.value.value);
break;
case 'boolean':
query.push('`' + rule.columnType.column + '` = ?');
query.push(prefix + '`' + rule.columnType.column + '` = ?');
values.push(rule.value.value);
break;
case 'number':
if (rule.value.range) {
if (rule.value.start) {
query.push('`' + rule.columnType.column + '` >= ?');
query.push(prefix + '`' + rule.columnType.column + '` >= ?');
values.push(rule.value.start);
}
if (rule.value.end) {
query.push('`' + rule.columnType.column + '` < ?');
query.push(prefix + '`' + rule.columnType.column + '` < ?');
values.push(rule.value.end);
}
} else {
query.push('`' + rule.columnType.column + '` = ?');
query.push(prefix + '`' + rule.columnType.column + '` = ?');
values.push(rule.value.value);
}
break;
@ -565,13 +567,13 @@ module.exports.getQuery = (id, callback) => {
if (rule.value.range) {
let start = rule.value.start || '01-01';
let end = rule.value.end || '12-31';
query.push('`' + rule.columnType.column + '` >= ?');
query.push(prefix + '`' + rule.columnType.column + '` >= ?');
values.push(getDate('2000-' + start));
query.push('`' + rule.columnType.column + '` < ?');
query.push(prefix + '`' + rule.columnType.column + '` < ?');
values.push(getDate('2000-' + end, true));
} else {
query.push('`' + rule.columnType.column + '` >= ?');
query.push('`' + rule.columnType.column + '` < ?');
query.push(prefix + '`' + rule.columnType.column + '` >= ?');
query.push(prefix + '`' + rule.columnType.column + '` < ?');
values.push(getDate('2000-' + rule.value.value));
values.push(getDate('2000-' + rule.value.value, true));
}
@ -579,16 +581,16 @@ module.exports.getQuery = (id, callback) => {
case 'date':
if (rule.value.range) {
if (rule.value.start) {
query.push('`' + rule.columnType.column + '` >= ?');
query.push(prefix + '`' + rule.columnType.column + '` >= ?');
values.push(getDate(rule.value.start));
}
if (rule.value.end) {
query.push('`' + rule.columnType.column + '` < ?');
query.push(prefix + '`' + rule.columnType.column + '` < ?');
values.push(getDate(rule.value.end, true));
}
} else {
query.push('`' + rule.columnType.column + '` >= ?');
query.push('`' + rule.columnType.column + '` < ?');
query.push(prefix + '`' + rule.columnType.column + '` >= ?');
query.push(prefix + '`' + rule.columnType.column + '` < ?');
values.push(getDate(rule.value.value));
values.push(getDate(rule.value.value, true));
}
@ -612,7 +614,7 @@ module.exports.subscribers = (id, onlySubscribed, callback) => {
if (!segment) {
return callback(new Error('Segment not found'));
}
module.exports.getQuery(id, (err, queryData) => {
module.exports.getQuery(id, false, (err, queryData) => {
if (err) {
return callback(err);
}

2
lib/models/subscriptions.js
View file

@ -122,7 +122,7 @@ module.exports.filter = (listId, request, columns, segmentId, callback) => {
};
if (segmentId) {
segments.getQuery(segmentId, (err, queryData) => {
segments.getQuery(segmentId, false, (err, queryData) => {
if (err) {
return callback(err);
}

1
package.json
View file

@ -42,6 +42,7 @@
"escape-html": "^1.0.3",
"express": "^4.13.4",
"express-session": "^1.13.0",
"faker": "^3.1.0",
"feedparser": "^1.1.4",
"geoip-ultralight": "^0.1.3",
"handlebars": "^4.0.5",

2
routes/lists.js
View file

@ -212,7 +212,6 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
list.imports = imports.map((entry, i) => {
entry.index = i + 1;
entry.processed = humanize.numberFormat(entry.processed, 0);
entry.importType = entry.type === 1 ? 'Subscribe' : 'Unsubscribe';
switch (entry.status) {
case 0:
@ -234,6 +233,7 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
entry.created = entry.created && entry.created.toISOString();
entry.finished = entry.finished && entry.finished.toISOString();
entry.updated = entry.processed - entry.new;
entry.processed = humanize.numberFormat(entry.processed, 0);
return entry;
});
list.csrfToken = req.csrfToken();

141
services/sender.js
View file

@ -16,7 +16,44 @@ let url = require('url');
let htmlToText = require('html-to-text');
let request = require('request');
// to speed things up fetch several unsent messages and store these into a cache
let fetchCache = [];
function findUnsent(callback) {
let returnUnsent = (row, campaign) => {
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
let subscription = tools.convertKeys(row);
let query = 'INSERT INTO `campaign__' + campaign.id + '` (list, segment, subscription) VALUES(?, ?,?)';
connection.query(query, [campaign.list, campaign.segment, subscription.id], (err, result) => {
connection.release();
if (err) {
if (err.code === 'ER_DUP_ENTRY') {
// race condition, try next one
return findUnsent(callback);
}
return callback(err);
}
subscription.campaign = campaign.id;
callback(null, {
id: result.insertId,
listId: campaign.list,
campaignId: campaign.id,
subscription
});
});
});
};
if (fetchCache.length) {
let cached = fetchCache.shift();
return returnUnsent(cached.row, cached.campaign);
}
db.getConnection((err, connection) => {
if (err) {
return callback(err);
@ -24,13 +61,13 @@ function findUnsent(callback) {
// Find "normal" campaigns. Ignore RSS and drip campaigns at this point
let query = 'SELECT `id`, `list`, `segment` FROM `campaigns` WHERE `status`=? AND (`scheduled` IS NULL OR `scheduled` <= NOW()) AND `type` IN (?, ?) LIMIT 1';
connection.query(query, [2, 1, 3], (err, rows) => {
connection.release();
if (err) {
connection.release();
return callback(err);
}
if (!rows || !rows.length) {
connection.release();
return callback(null, false);
}
@ -44,7 +81,7 @@ function findUnsent(callback) {
values: []
});
}
segments.getQuery(segmentId, next);
segments.getQuery(segmentId, 'subscription', next);
};
getSegmentQuery(campaign.segment, (err, queryData) => {
@ -52,15 +89,27 @@ function findUnsent(callback) {
return callback(err);
}
let tryNext = () => {
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
// TODO: Add support for localized sending time. In this case campaign messages are
// not sent before receiver's local time reaches defined time
// SELECT * FROM subscription__1 LEFT JOIN tzoffset ON tzoffset.tz=subscription__1.tz WHERE NOW() + INTERVAL IFNULL(`offset`,0) MINUTE >= localtime
let query = 'SELECT * FROM `subscription__' + campaign.list + '` subscription WHERE status=1 ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' AND NOT EXISTS (SELECT 1 FROM `campaign__' + campaign.id + '` campaign WHERE campaign.list = ? AND campaign.segment = ? AND campaign.subscription = subscription.id) LIMIT 1';
let query;
let values;
connection.query(query, queryData.values.concat([campaign.list, campaign.segment]), (err, rows) => {
// NOT IN
query = 'SELECT * FROM `subscription__' + campaign.list + '` AS subscription WHERE status=1 ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' AND id NOT IN (SELECT subscription FROM `campaign__' + campaign.id + '` campaign WHERE campaign.list = ? AND campaign.segment = ? AND campaign.subscription = subscription.id) LIMIT 100';
values = queryData.values.concat([campaign.list, campaign.segment]);
// LEFT JOIN / IS NULL
//query = 'SELECT subscription.* FROM `subscription__' + campaign.list + '` AS subscription LEFT JOIN `campaign__' + campaign.id + '` AS campaign ON campaign.list = ? AND campaign.segment = ? AND campaign.subscription = subscription.id WHERE subscription.status=1 ' + (queryData.where ? 'AND (' + queryData.where + ') ' : '') + 'AND campaign.id IS NULL LIMIT 100';
//values = [campaign.list, campaign.segment].concat(queryData.values);
connection.query(query, values, (err, rows) => {
if (err) {
connection.release();
return callback(err);
@ -73,30 +122,19 @@ function findUnsent(callback) {
return callback(null, false);
});
}
connection.release();
let subscription = tools.convertKeys(rows[0]);
let query = 'INSERT INTO `campaign__' + campaign.id + '` (list, segment, subscription) VALUES(?, ?,?)';
connection.query(query, [campaign.list, campaign.segment, subscription.id], (err, result) => {
if (err) {
if (err.code === 'ER_DUP_ENTRY') {
// race condition, try next one
return tryNext();
}
connection.release();
return callback(err);
}
connection.release();
subscription.campaign = campaign.id;
callback(null, {
id: result.insertId,
listId: campaign.list,
campaignId: campaign.id,
subscription
rows.forEach(row => {
fetchCache.push({
row,
campaign
});
});
return findUnsent(callback);
});
};
tryNext();
});
});
});
});
@ -294,41 +332,52 @@ let sendLoop = () => {
return;
}
// send the message
mailer.transport.sendMail(mail, (err, info) => {
if (err) {
log.error('Mail', err.stack);
}
let tryCount = 0;
let trySend = () => {
tryCount++;
let status = err ? 2 : 1;
let response = err && (err.response || err.message) || info.response;
let responseId = response.split(/\s+/).pop();
db.getConnection((err, connection) => {
// send the message
mailer.transport.sendMail(mail, (err, info) => {
if (err) {
log.error('Mail', err.stack);
return;
if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) {
// temporary error, try again
return setTimeout(trySend, tryCount * 1000);
}
}
let query = 'UPDATE `campaigns` SET `delivered`=`delivered`+1 ' + (status === 2 ? ', `bounced`=`bounced`+1 ' : '') + ' WHERE id=? LIMIT 1';
connection.query(query, [message.campaignId], err => {
let status = err ? 2 : 1;
let response = err && (err.response || err.message) || info.response;
let responseId = response.split(/\s+/).pop();
db.getConnection((err, connection) => {
if (err) {
log.error('Mail', err.stack);
return;
}
let query = 'UPDATE `campaign__' + message.campaignId + '` SET status=?, response=?, response_id=?, updated=NOW() WHERE id=? LIMIT 1';
connection.query(query, [status, response, responseId, message.id], err => {
connection.release();
let query = 'UPDATE `campaigns` SET `delivered`=`delivered`+1 ' + (status === 2 ? ', `bounced`=`bounced`+1 ' : '') + ' WHERE id=? LIMIT 1';
connection.query(query, [message.campaignId], err => {
if (err) {
log.error('Mail', err.stack);
} else {
// log.verbose('Mail', 'Message sent and status updated for %s', message.subscription.cid);
}
let query = 'UPDATE `campaign__' + message.campaignId + '` SET status=?, response=?, response_id=?, updated=NOW() WHERE id=? LIMIT 1';
connection.query(query, [status, response, responseId, message.id], err => {
connection.release();
if (err) {
log.error('Mail', err.stack);
} else {
// log.verbose('Mail', 'Message sent and status updated for %s', message.subscription.cid);
}
});
});
});
});
});
};
setImmediate(trySend);
setImmediate(getNext);
});
});

40
services/test-server.js
View file

@ -3,14 +3,18 @@
let log = require('npmlog');
let config = require('config');
let crypto = require('crypto');
let humanize = require('humanize');
let SMTPServer = require('smtp-server').SMTPServer;
let totalMessages = 0;
let received = 0;
// Setup server
let server = new SMTPServer({
// log to console
logger: false,
logger: config.testserver.logger,
// not required but nice-to-have
banner: 'Welcome to My Awesome SMTP Server',
@ -19,25 +23,18 @@ let server = new SMTPServer({
disabledCommands: ['STARTTLS'],
// By default only PLAIN and LOGIN are enabled
authMethods: ['PLAIN', 'LOGIN', 'CRAM-MD5'],
authMethods: ['PLAIN', 'LOGIN'],
// Accept messages up to 10 MB
size: 10 * 1024 * 1024,
// Setup authentication
// Allow only users with username 'testuser' and password 'testpass'
onAuth: (auth, session, callback) => {
let username = 'testuser';
let password = 'testpass';
let username = config.testserver.username;
let password = config.testserver.password;
// check username and password
if (auth.username === username &&
(
auth.method === 'CRAM-MD5' ?
auth.validatePassword(password) : // if cram-md5, validate challenge response
auth.password === password // for other methods match plaintext passwords
)
) {
if (auth.username === username && auth.password === password) {
return callback(null, {
user: 'userdata' // value could be an user id, or an user object etc. This value can be accessed from session.user afterwards
});
@ -87,19 +84,34 @@ let server = new SMTPServer({
err.responseCode = 552;
return callback(err);
}
received++;
callback(null, 'Message queued as ' + hash.digest('hex')); // accept the message once the stream is ended
});
}
});
server.on('error', err => {
log.error('TESTSERV', err.stack);
log.error('Test SMTP', err.stack);
});
module.exports = callback => {
if (config.testserver.enabled) {
server.listen(config.testserver.port, config.testserver.host, () => {
log.info('TESTSERV', 'Server listening on port %s', config.testserver.port);
log.info('Test SMTP', 'Server listening on port %s', config.testserver.port);
setInterval(() => {
if (received) {
totalMessages += received;
log.verbose(
'Test SMTP',
'Received %s new message%s in last 60 sec. (total %s messages)',
humanize.numberFormat(received, 0), received === 1 ? '' : 's',
humanize.numberFormat(totalMessages, 0)
);
received = 0;
}
}, 60 * 1000);
setImmediate(callback);
});
} else {

28
setup/fakedata.js Normal file
View file

@ -0,0 +1,28 @@
'use strict';
let faker = require('faker');
let accounts = 100 * 1000;
let row = 0;
let getNext = () => {
let firstName = faker.name.firstName(); // Rowan Nikolaus
let lastName = faker.name.lastName(); // Rowan Nikolaus
let email = faker.internet.email(firstName, lastName); // Kassandra.Haley@erich.biz
let subscriber = {
firstName,
lastName,
email,
company: faker.company.companyName(),
phone: faker.phone.phoneNumber()
};
process.stdout.write('\n' + Object.keys(subscriber).map(key => JSON.stringify(subscriber[key])).join(','));
if (++row < accounts) {
setImmediate(getNext);
}
};
process.stdout.write('First name,Last name,E-Mail,Company,Phone number');
getNext();