public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Various fixes.

Browse source
This commit is contained in:
Tomas Bures 2018-12-23 19:27:29 +00:00
parent dd9b8b464a
commit 83ce716d94
21 changed files with 99 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

4
server/lib/builtin-zone-mta.js
View file

@ -10,9 +10,9 @@ const crypto = require('crypto');
let zoneMtaProcess;
const zoneMtaDir = path.join(__dirname, '..', '..', 'zone-mta');
const zoneMtaBuiltingConfig = path.join(zoneMtaDir, 'config', 'builtin-zonemta.json')
const zoneMtaBuiltingConfig = path.join(zoneMtaDir, 'config', 'builtin-zonemta.json');
const password = crypto.randomBytes(20).toString('hex').toLowerCase();
const password = process.env.BUILTIN_ZONE_MTA_PASSWORD || crypto.randomBytes(20).toString('hex').toLowerCase();
function getUsername() {
return 'mailtrain';

7
server/lib/mailers.js
View file

@ -95,11 +95,18 @@ async function _sendMail(transport, mail, template) {
}
async function _sendTransactionalMail(transport, mail, template) {
const sendConfiguration = transport.mailer.sendConfiguration;
if (!mail.headers) {
mail.headers = {};
}
mail.headers['X-Sending-Zone'] = 'transactional';
mail.from = {
name: sendConfiguration.from_name,
address: sendConfiguration.from_email
};
const htmlRenderer = await tools.getTemplate(template.html, template.locale);
if (htmlRenderer) {

6
server/lib/senders.js
View file

@ -5,6 +5,7 @@ const log = require('./log');
const path = require('path');
const knex = require('./knex');
const {CampaignStatus} = require('../../shared/campaigns');
const builtinZoneMta = require('./builtin-zone-mta');
let messageTid = 0;
let senderProcess;
@ -16,7 +17,10 @@ function spawn(callback) {
.then(() => {
senderProcess = fork(path.join(__dirname, '..', 'services', 'sender-master.js'), [], {
cwd: path.join(__dirname, '..'),
env: {NODE_ENV: process.env.NODE_ENV}
env: {
NODE_ENV: process.env.NODE_ENV,
BUILTIN_ZONE_MTA_PASSWORD: builtinZoneMta.getPassword()
}
});
senderProcess.on('message', msg => {

6
server/lib/subscription-mail-helpers.js
View file

@ -110,7 +110,7 @@ async function _sendMail(list, email, template, locale, subjectKey, relativeUrls
const data = {
title: list.name,
homepage: configItems.defaultHomepage || getTrustedUrl(),
contactAddress: list.from_email || configItems.adminEmail,
contactAddress: list.contact_email || configItems.adminEmail,
};
for (let relativeUrlKey in relativeUrls) {
@ -140,10 +140,6 @@ async function _sendMail(list, email, template, locale, subjectKey, relativeUrls
if (list.send_configuration) {
const mailer = await mailers.getOrCreateMailer(list.send_configuration);
await mailer.sendTransactionalMail({
from: {
name: configItems.defaultFrom,
address: configItems.defaultAddress
},
to: {
name: getDisplayName(flds, subscription),
address: email

36
server/models/campaigns.js
View file

@ -324,6 +324,7 @@ async function rawGetByTx(tx, key, id) {
'campaigns.id', 'campaigns.cid', 'campaigns.name', 'campaigns.description', 'campaigns.namespace', 'campaigns.status', 'campaigns.type', 'campaigns.source',
'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject_override',
'campaigns.data', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled', 'campaigns.unsubscribe_url', 'campaigns.scheduled',
'campaigns.delivered', 'campaigns.unsubscribed', 'campaigns.bounced', 'campaigns.complained', 'campaigns.blacklisted', 'campaigns.opened', 'campaigns.clicks',
knex.raw(`GROUP_CONCAT(CONCAT_WS(\':\', campaign_lists.list, campaign_lists.segment) ORDER BY campaign_lists.id SEPARATOR \';\') as lists`)
])
.first();
@ -361,18 +362,12 @@ async function getByIdTx(tx, context, id, withPermissions = true, content = Cont
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id, true);
const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id);
if (unsentQryGen) {
const res = await unsentQryGen(tx).count('* AS subscriptionsToSend').first();
entity.subscriptionsToSend = res.subscriptionsToSend;
}
const totalQryGen = await getSubscribersQueryGeneratorTx(tx, id, false);
if (totalQryGen) {
const res = await totalQryGen(tx).count('* AS subscriptionsTotal').first();
entity.subscriptionsTotal = res.subscriptionsTotal;
}
} else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
delete entity.data.sourceCustom;
@ -765,7 +760,7 @@ async function updateMessageResponse(context, message, response, responseId) {
});
}
async function getSubscribersQueryGeneratorTx(tx, campaignId, onlyUnsent) {
async function getSubscribersQueryGeneratorTx(tx, campaignId) {
/*
This is supposed to produce queries like this:
@ -812,7 +807,7 @@ async function getSubscribersQueryGeneratorTx(tx, campaignId, onlyUnsent) {
if (subsQrys.length > 0) {
let subsQry;
const unsentWhere = onlyUnsent ? ' where `sent` = false' : '';
const unsentWhere = ' where `sent` = false';
if (subsQrys.length === 1) {
const subsUnionSql = '(select `email`, `campaign_list_id`, `sent` from (' + subsQrys[0].sql + ') as `pending_subscriptions_all`' + unsentWhere + ') as `pending_subscriptions`'
@ -905,30 +900,12 @@ async function disable(context, campaignId) {
}
async function getStatisticsOverview(context, id) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
const stats = await tx('campaigns').where('id', id).select(['delivered', 'unsubscribed', 'bounced', 'complained', 'blacklisted', 'opened', 'clicks']).first();
const totalQryGen = await getSubscribersQueryGeneratorTx(tx, id, false);
if (totalQryGen) {
const res = await totalQryGen(tx).count('* AS subscriptionsTotal').first();
stats.total = res.subscriptionsTotal;
} else {
stats.total = 0;
}
return stats;
});
}
async function getStatisticsOpened(context, id) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
const devices = await tx('campaign_links').where('campaign', id).groupBy('device_type').select('device_type AS key').count('* as count');
const countries = await tx('campaign_links').where('campaign', id).groupBy('country').select('country AS key').count('* as count');
const devices = await tx('campaign_links').where('campaign', id).where('link', LinkId.OPEN).groupBy('device_type').select('device_type AS key').count('* as count');
const countries = await tx('campaign_links').where('campaign', id).where('link', LinkId.OPEN).groupBy('country').select('country AS key').count('* as count');
return {
devices,
@ -976,5 +953,4 @@ module.exports.disable = disable;
module.exports.rawGetByTx = rawGetByTx;
module.exports.getTrackingSettingsByCidTx = getTrackingSettingsByCidTx;
module.exports.getStatisticsOverview = getStatisticsOverview;
module.exports.getStatisticsOpened = getStatisticsOpened;

9
server/models/links.js
View file

@ -32,7 +32,10 @@ async function countLink(remoteIp, userAgent, campaignCid, listCid, subscription
const subscription = await subscriptions.getByCidTx(tx, contextHelpers.getAdminContext(), list.id, subscriptionCid);
const country = geoip.lookupCountry(remoteIp) || null;
const device = uaParser(userAgent, { unknownUserAgentDeviceType: 'desktop', emptyUserAgentDeviceType: 'desktop' });
const device = uaParser(userAgent, {
unknownUserAgentDeviceType: 'desktop',
emptyUserAgentDeviceType: 'desktop'
});
const now = new Date();
const _countLink = async (clickLinkId, incrementOnDup) => {
@ -66,7 +69,6 @@ async function countLink(remoteIp, userAgent, campaignCid, listCid, subscription
};
// Update opened and click timestamps
const latestUpdates = {};
@ -82,7 +84,6 @@ async function countLink(remoteIp, userAgent, campaignCid, listCid, subscription
await tx(subscriptions.getSubscriptionTableName(list.id)).update(latestUpdates).where('id', subscription.id);
}
// Update clicks
if (linkId > LinkId.GENERAL_CLICK && !campaign.click_tracking_disabled) {
await tx('links').increment('hits').where('id', linkId);
@ -125,6 +126,8 @@ async function addOrGet(campaignId, url) {
id: ids[0],
cid
};
} else {
return link;
}
});
}

25
server/models/reports.js
View file

@ -255,31 +255,24 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
} else {
return knex.raw(subsSql, subsBindings);
}
}
};
if (subsQrys.length === 1) {
subsSql = subsQrys[0].sql;
subsBindings = subsQrys[0].bindings;
if (asStream) {
return await applyUnionQryFn(subsSql, subsBindings).stream();
} else {
return await applyUnionQryFn(subsSql, subsBindings);
}
} else {
subsSql = subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ');
subsBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
}
if (asStream) {
return applyUnionQryFn(subsSql, subsBindings).stream();
if (asStream) {
return applyUnionQryFn(subsSql, subsBindings).stream();
} else {
const res = await applyUnionQryFn(subsSql, subsBindings);
if (res[0] && Array.isArray(res[0])) {
return res[0]; // UNION ALL generates an array with result and schema
} else {
const res = await applyUnionQryFn(subsSql, subsBindings);
if (res[0] && Array.isArray(res[0])) {
return res[0]; // UNION ALL generates an array with result and schema
} else {
return res;
}
return res;
}
}

3
server/models/users.js
View file

@ -304,9 +304,6 @@ async function sendPasswordReset(locale, usernameOrEmail) {
const mailer = await mailers.getOrCreateMailer();
await mailer.sendTransactionalMail({
from: {
address: adminEmail
},
to: {
address: user.email
},

3
server/routes/links.js
View file

@ -13,9 +13,10 @@ router.getAsync('/:campaign/:list/:subscription/:link', async (req, res) => {
if (link) {
// In Mailtrain v1 we would do the URL expansion here based on merge tags. We don't do it here anymore. Instead, the URLs are expanded when message is sent out (in links.updateLinks)
res.redirect(link.url);
res.redirect(302, link.url);
await links.countLink(req.ip, req.headers['user-agent'], req.params.campaign, req.params.list, req.params.subscription, link.id);
} else {
log.error('Redirect', 'Unresolved URL: <%s>', req.url);
throw new interoperableErrors.NotFoundError('Oops, we couldn\'t find a link for the URL you clicked');

4
server/routes/rest/campaigns.js
View file

@ -94,10 +94,6 @@ router.postAsync('/campaign-disable/:campaignId', passport.loggedIn, passport.cs
return res.json(await campaigns.disable(req.context, castToInteger(req.params.campaignId), null));
});
router.getAsync('/campaign-statistics/:campaignId/overview', passport.loggedIn, async (req, res) => {
return res.json(await campaigns.getStatisticsOverview(req.context, castToInteger(req.params.campaignId)));
});
router.getAsync('/campaign-statistics/:campaignId/opened', passport.loggedIn, async (req, res) => {
return res.json(await campaigns.getStatisticsOpened(req.context, castToInteger(req.params.campaignId)));
});

4
server/routes/subscription.js
View file

@ -670,8 +670,8 @@ async function webNotice(type, req, res) {
const data = {
title: list.name,
homepage: configItems.defaultHomepage || getTrustedUrl(),
contactAddress: list.from_email || configItems.adminEmail,
homepage: list.homepage || configItems.defaultHomepage || getTrustedUrl(),
contactAddress: list.contact_email || configItems.adminEmail,
template: {
template: 'subscription/web-' + type + '-notice.mjml.hbs',
layout: 'subscription/layout.mjml.hbs',

37
server/services/sender-master.js
View file

@ -8,6 +8,7 @@ const knex = require('../lib/knex');
const {CampaignStatus, CampaignType} = require('../../shared/campaigns');
const { enforce } = require('../lib/helpers');
const campaigns = require('../models/campaigns');
const builtinZoneMta = require('../lib/builtin-zone-mta');
let messageTid = 0;
const workerProcesses = new Map();
@ -24,6 +25,7 @@ const workerBatchSize = 100;
const messageQueue = new Map(); // campaignId -> [{listId, email}]
const messageQueueCont = new Map(); // campaignId -> next batch callback
const campaignFinishCont = new Map(); // campaignId -> worker finished callback
const workAssignment = new Map(); // workerId -> { campaignId, subscribers: [{listId, email}] }
@ -32,6 +34,8 @@ let queuedLastId = 0;
function messagesProcessed(workerId) {
const wa = workAssignment.get(workerId);
workAssignment.delete(workerId);
idleWorkers.push(workerId);
@ -40,6 +44,11 @@ function messagesProcessed(workerId) {
setImmediate(workerSchedulerCont);
workerSchedulerCont = null;
}
if (campaignFinishCont.has(wa.campaignId)) {
setImmediate(campaignFinishCont.get(wa.campaignId));
campaignFinishCont.delete(wa.campaignId);
}
}
async function scheduleWorkers() {
@ -78,8 +87,8 @@ async function scheduleWorkers() {
workAssignment.set(workerId, {campaignId, subscribers});
if (queue.length === 0 && messageQueueCont.has(campaignId)) {
const scheduleMessages = messageQueueCont.get(campaignId);
setImmediate(scheduleMessages);
setImmediate(messageQueueCont.get(campaignId));
messageQueueCont.delete(campaignId);
}
sendToWorker(workerId, 'process-messages', {
@ -99,9 +108,24 @@ async function scheduleWorkers() {
}
async function processCampaign(campaignId) {
async function finish() {
let workerRunning = false;
for (const wa of workAssignment.values()) {
if (wa.campaignId === campaignId) {
workerRunning = true;
}
}
if (workerRunning) {
const workerFinished = new Promise(resolve => {
campaignFinishCont.set(campaignId, resolve);
});
await workerFinished;
setImmediate(finish);
}
await knex('campaigns').where('id', campaignId).update({status: CampaignStatus.FINISHED});
messageQueue.delete(campaignId);
}
@ -120,7 +144,7 @@ async function processCampaign(campaignId) {
let qryGen;
await knex.transaction(async tx => {
qryGen = await campaigns.getSubscribersQueryGeneratorTx(tx, campaignId, true);
qryGen = await campaigns.getSubscribersQueryGeneratorTx(tx, campaignId);
});
if (qryGen) {
@ -262,7 +286,10 @@ async function spawnWorker(workerId) {
const senderProcess = fork(path.join(__dirname, 'sender-worker.js'), [workerId], {
cwd: path.join(__dirname, '..'),
env: {NODE_ENV: process.env.NODE_ENV}
env: {
NODE_ENV: process.env.NODE_ENV,
BUILTIN_ZONE_MTA_PASSWORD: builtinZoneMta.getPassword()
}
});
senderProcess.on('message', msg => {