public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Work in progress on refactoring all mail sending to use the message sender an sender workers.

Browse source 
Some fixes related to subscriptions and password reset.
This commit is contained in:
Tomas Bures 2019-06-30 10:47:09 +02:00
parent 4e9f6bd57b
commit 450b930cc5
8 changed files with 95 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

1
server/services/feedcheck.js
View file

@ -5,7 +5,6 @@ const log = require('../lib/log');
const knex = require('../lib/knex');
const feedparser = require('feedparser-promised');
const { CampaignType, CampaignStatus, CampaignSource } = require('../../shared/campaigns');
const util = require('util');
const campaigns = require('../models/campaigns');
const contextHelpers = require('../lib/context-helpers');
require('../lib/fork');

55
server/services/sender-master.js
View file

@ -175,6 +175,10 @@ async function workersLoop() {
return idleWorkers.shift();
}
function cancelWorker(workerId) {
idleWorkers.push(workerId);
}
function selectNextTask() {
const allocationMap = new Map();
const allocation = [];
@ -251,11 +255,10 @@ async function workersLoop() {
while (true) {
const workerId = await getAvailableWorker();
const task = selectNextTask();
if (task) {
const workerId = await getAvailableWorker();
const attrName = task.attrName;
const sendConfigurationId = task.sendConfigurationId;
const sendConfigurationStatus = getSendConfigurationStatus(sendConfigurationId);
@ -280,7 +283,9 @@ async function workersLoop() {
[attrName]: task.id,
messages
});
} else {
cancelWorker(workerId);
await notifier.waitFor('workAvailable');
}
}
@ -394,7 +399,7 @@ async function scheduleCampaigns() {
campaignSchedulerRunning = true;
try {
// finish old campaigns
// Finish old campaigns
const nowDate = new Date();
const now = nowDate.valueOf();
@ -405,6 +410,22 @@ async function scheduleCampaigns() {
.where('campaigns.start_at', '<', expirationThreshold)
.update({status: CampaignStatus.FINISHED});
// Empty message queues for PAUSING campaigns. A pausing campaign typically waits for campaignMessageQueueEmpty before it can check for PAUSING
// We speed this up by discarding messages in the message queue of the campaign.
const pausingCampaigns = await knex('campaigns')
.whereIn('campaigns.type', [CampaignType.REGULAR, CampaignType.RSS_ENTRY])
.where('campaigns.status', CampaignStatus.PAUSING)
.select(['id'])
.forUpdate();
for (const cpg of pausingCampaigns) {
const campaignId = cpg.id;
const queue = campaignMessageQueue.get(campaignId);
queue.splice(0);
notifier.notify(`campaignMessageQueueEmpty:${campaignId}`);
}
while (true) {
let campaignId = 0;
const postponedSendConfigurationIds = getPostponedSendConfigurationIds();
@ -513,30 +534,26 @@ async function processQueuedBySendConfiguration(sendConfigurationId) {
const expirationThresholds = getExpirationThresholds();
const expirationCounters = {};
for (const type of Object.keys(expirationThresholds)) {
for (const type in expirationThresholds) {
expirationCounters[type] = 0;
}
for (const row of rows) {
for (const type of Object.keys(expirationThresholds)) {
if (row.type === type) {
const expirationThreshold = expirationThresholds[type];
const expirationThreshold = expirationThresholds[row.type];
if (row.created < expirationThreshold.threshold) {
expirationCounters[type] += 1;
await knex('queued').where('id', row.id).del();
if (row.created < expirationThreshold.threshold) {
expirationCounters[row.type] += 1;
await knex('queued').where('id', row.id).del();
} else {
row.data = JSON.parse(row.data);
msgQueue.push({
queuedMessage: row
});
}
}
} else {
row.data = JSON.parse(row.data);
msgQueue.push({
queuedMessage: row
});
}
}
for (const type of Object.keys(expirationThresholds)) {
for (const type in expirationThresholds) {
const expirationThreshold = expirationThresholds[type];
if (expirationCounters[type] > 0) {
log.warn('Senders', `Discarded ${expirationCounters[type]} expired ${expirationThreshold.title} message(s).`);
@ -568,7 +585,7 @@ async function scheduleQueued() {
// prune old messages
const expirationThresholds = getExpirationThresholds();
for (const type of Object.keys(expirationThresholds)) {
for (const type in expirationThresholds) {
const expirationThreshold = expirationThresholds[type];
const expiredCount = await knex('queued')

43
server/services/sender-worker.js
View file

@ -17,25 +17,25 @@ async function processCampaignMessages(campaignId, messages) {
running = true;
const cs = new MessageSender();
const cs = new messageSender.MessageSender();
await cs.initByCampaignId(campaignId);
let withErrors = false;
for (const msgData of messages) {
for (const msg of messages) {
try {
await cs.sendRegularMessage(msgData.listId, msgData.email);
await cs.sendRegularMessage(msg.listId, msg.email);
log.verbose('Senders', 'Message sent and status updated for %s:%s', msgData.listId, msgData.email);
log.verbose('Senders', 'Message sent and status updated for %s:%s', msg.listId, msg.email);
} catch (err) {
if (err instanceof mailers.SendConfigurationError) {
log.error('Senders', `Sending message to ${msgData.listId}:${msgData.email} failed with error: ${err.message}. Will retry the message if within retention interval.`);
log.error('Senders', `Sending message to ${msg.listId}:${msg.email} failed with error: ${err.message}. Will retry the message if within retention interval.`);
withErrors = true;
break;
} else {
log.error('Senders', `Sending message to ${msgData.listId}:${msgData.email} failed with error: ${err.message}. Dropping the message.`);
log.error('Senders', `Sending message to ${msg.listId}:${msg.email} failed with error: ${err.message}.`);
log.verbose(err.stack);
}
}
@ -56,19 +56,40 @@ async function processQueuedMessages(sendConfigurationId, messages) {
let withErrors = false;
for (const msgData of messages) {
const queuedMessage = msgData.queuedMessage;
for (const msg of messages) {
const queuedMessage = msg.queuedMessage;
const msgData = queuedMessage.data;
let target = '';
if (msgData.listId && msgData.subscriptionId) {
target = `${msgData.listId}:${msgData.subscriptionId}`;
} else if (msgData.to) {
if (msgData.to.name && msgData.to.address) {
target = `${msgData.to.name} <${msgData.to.address}>`;
} else if (msgData.to.address) {
target = msgData.to.address;
} else {
target = msgData.to.toString();
}
}
try {
await messageSender.sendQueuedMessage(queuedMessage);
log.verbose('Senders', 'Message sent and status updated for %s:%s', queuedMessage.list, queuedMessage.subscription);
log.verbose('Senders', `Message sent and status updated for ${target}`);
} catch (err) {
if (err instanceof mailers.SendConfigurationError) {
log.error('Senders', `Sending message to ${queuedMessage.list}:${queuedMessage.subscription} failed with error: ${err.message}. Will retry the message if within retention interval.`);
log.error('Senders', `Sending message to ${target} failed with error: ${err.message}. Will retry the message if within retention interval.`);
withErrors = true;
break;
} else {
log.error('Senders', `Sending message to ${queuedMessage.list}:${queuedMessage.subscription} failed with error: ${err.message}. Dropping the message.`);
log.error('Senders', `Sending message to ${target} failed with error: ${err.message}. Dropping the message.`);
log.verbose(err.stack);
try {
await messageSender.dropQueuedMessage(queuedMessage);
} catch (err) {
log.error(err.stack);
}
}
}
}