Implementation of archive route. Simplified from v1. Not tested.

This commit is contained in:
Tomas Bures 2018-09-22 18:12:22 +02:00
parent a9e1700dbe
commit dda95ecdb3
9 changed files with 320 additions and 272 deletions

View file

@ -3,8 +3,6 @@
const config = require('config');
const log = require('npmlog');
const _ = require('./lib/translate')._;
const express = require('express');
const bodyParser = require('body-parser');
const path = require('path');
@ -27,6 +25,7 @@ const subscription = require('./routes/subscription');
const mosaico = require('./routes/mosaico');
const files = require('./routes/files');
const links = require('./routes/links');
const archive = require('./routes/archive');
const namespacesRest = require('./routes/rest/namespaces');
const sendConfigurationsRest = require('./routes/rest/send-configurations');
@ -217,6 +216,7 @@ function createApp(appType) {
if (appType === AppType.PUBLIC) {
useWith404Fallback('/subscription', subscription);
useWith404Fallback('/links', links);
useWith404Fallback('/archive', archive);
}
if (appType === AppType.TRUSTED || appType === AppType.SANDBOXED) {

View file

@ -68,7 +68,11 @@ class TestUser extends Component {
if (this.isFormWithoutErrors()) {
const data = this.getFormValues();
console.log(this.props.entity);
console.log(data);
// FIXME - navigate to campaign preview
// window.location =
} else {
this.showFormValidation();
}

268
lib/campaign-sender.js Normal file
View file

@ -0,0 +1,268 @@
'use strict';
const config = require('config');
const mailers = require('../lib/mailers');
const knex = require('../lib/knex');
const subscriptions = require('../models/subscriptions');
const contextHelpers = require('../lib/context-helpers');
const campaigns = require('../models/campaigns');
const templates = require('../models/templates');
const lists = require('../models/lists');
const fields = require('../models/fields');
const sendConfigurations = require('../models/send-configurations');
const links = require('../models/links');
const {CampaignSource} = require('../shared/campaigns');
const {SubscriptionStatus} = require('../shared/lists');
const tools = require('../lib/tools');
const request = require('request-promise');
const files = require('../models/files');
const htmlToText = require('html-to-text');
const {getPublicUrl} = require('../lib/urls');
const blacklist = require('../models/blacklist');
const libmime = require('libmime');
class CampaignSender {
constructor() {
}
async init(settings) {
this.listsById = Map(); // listId -> list
this.listsByCid = Map(); // listCid -> list
this.listsFieldsGrouped = Map(); // listId -> fieldsGrouped
this.attachments = [];
await knex.transaction(async tx => {
if (settings.campaignCid) {
this.campaign = await campaigns.rawGetByTx(tx, 'cid', settings.campaignCid);
} else {
this.campaign = await campaigns.rawGetByTx(tx, 'id', settings.campaignId);
}
this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration);
for (const listSpec of campaign.lists) {
const list = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list);
this.listsById.set(list.id) = list;
this.listsByCid.set(list.cid) = list;
this.listsFieldsGrouped.set(list.id) = await fields.listGroupedTx(tx, list.id);
}
if (campaign.source === CampaignSource.TEMPLATE) {
this.template = templates.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.data.sourceTemplate, false);
}
const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', this.campaign.id);
for (const attachment of attachments) {
this.attachments.push({
filename: attachment.originalname,
path: files.getFilePath('campaign', 'attachment', this.campaign.id, attachment.filename)
});
}
});
this.useVerp = config.verp.enabled && sendConfiguration.verp_hostname;
this.useVerpSenderHeader = useVerp && config.verp.disablesenderheader !== true;
}
async _getMessage(campaign, list, subscriptionGrouped, mergeTags, replaceDataImgs) {
let html = '';
let text = '';
let renderTags = false;
if (campaign.source === CampaignSource.URL) {
const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
for (const key in mergeTags) {
form[key] = mergeTags[key];
}
const response = await request.post({
uri: campaign.sourceUrl,
form,
resolveWithFullResponse: true
});
if (response.statusCode !== 200) {
throw new Error(`Received status code ${httpResponse.statusCode} from ${campaign.sourceUrl}`);
}
html = response.body;
text = '';
renderTags = false;
} else if (campaign.source === CampaignSource.CUSTOM || campaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN || campaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
html = campaign.data.sourceCustom.html;
text = campaign.data.sourceCustom.text;
renderTags = true;
} else if (campaign.source === CampaignSource.TEMPLATE) {
const template = this.template;
html = template.html;
text = template.text;
renderTags = true;
}
html = await links.updateLinks(campaign, list, subscriptionGrouped, mergeTags, html);
const attachments = this.attachments.slice();
if (replaceDataImgs) {
// replace data: images with embedded attachments
html = html.replace(/(<img\b[^>]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => {
let cid = shortid.generate() + '-attachments@' + campaign.address.split('@').pop();
attachments.push({
path: dataUri,
cid
});
return prefix + 'cid:' + cid;
});
}
const html = renderTags ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, html, false, true) : html;
const text = (text || '').trim()
? (renderTags ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, text) : text)
: htmlToText.fromString(html, {wordwrap: 130});
return {
html,
text,
attachments
};
}
async getMessage(listCid, subscriptionCid) {
const list = this.listsByCid.get(listCid);
const subscriptionGrouped = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid);
const flds = this.listsFieldsGrouped.get(list.id);
const campaign = this.campaign;
const mergeTags = fields.forHbsWithFieldsGrouped(flds, subscriptionGrouped);
return await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, false);
}
async sendMessage(listId, email) {
if (await blacklist.isBlacklisted(email)) {
return;
}
const list = this.listsById.get(list.id);
const subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, email);
const flds = this.listsFieldsGrouped.get(listId);
const campaign = this.campaign;
const mergeTags = fields.forHbsWithFieldsGrouped(flds, subscriptionGrouped);
const encryptionKeys = [];
for (const fld of flds) {
if (fld.type === 'gpg' && mergeTags[fld.key]) {
encryptionKeys.push(mergeTags[fld.key].trim());
}
}
const sendConfiguration = this.sendConfiguration;
const {html, text, attachments} = await this._getMessage(campaign, list, subscriptionGrouped, mergeTags, true);
const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
let listUnsubscribe = null;
if (!list.listunsubscribe_disabled) {
listUnsubscribe = campaign.unsubscribe_url
? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, campaign.unsubscribe_url)
: getPublicUrl('/subscription/' + list.cid + '/unsubscribe/' + subscriptionGrouped.subscription.cid);
}
const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
await mailer.throttleWait();
const getOverridable = key => {
if (sendConfiguration[key + '_overridable'] && this.campaign[key + '_override'] !== null) {
return campaign[key + '_override'];
} else {
return sendConfiguration[key];
}
}
const mail = {
from: {
name: getOverridable('from_name'),
address: getOverridable('from_email')
},
replyTo: getOverridable('reply_to'),
xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
to: {
name: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false, false),
address: subscriptionGrouped.email
},
sender: this.useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false,
envelope: this.useVerp ? {
from: campaignAddress + '@' + sendConfiguration.verp_hostname,
to: subscriptionGrouped.email
} : false,
headers: {
'x-fbl': campaignAddress,
// custom header for SparkPost
'x-msys-api': JSON.stringify({
campaign_id: campaignAddress
}),
// custom header for SendGrid
'x-smtpapi': JSON.stringify({
unique_args: {
campaign_id: campaignAddress
}
}),
// custom header for Mailgun
'x-mailgun-variables': JSON.stringify({
campaign_id: campaignAddress
}),
'List-ID': {
prepared: true,
value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>'
}
},
list: {
unsubscribe: listUnsubscribe
},
subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false, false),
html,
text,
attachments,
encryptionKeys
};
let status;
let response;
try {
const info = await mailer.sendMassMail(mail);
status = SubscriptionStatus.SUBSCRIBED;
response = info.response || info.messageId;
await knex('campaigns').where('id', campaign.id).increment('delivered');
} catch (err) {
status = SubscriptionStatus.BOUNCED;
response = err.response || err.message;
await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced');
}
const responseId = response.split(/\s+/).pop();
const now = new Date();
await knex('campaign_messages').insert({
campaign: this.campaign.id,
list: listId,
subscriptions: subscriptionGrouped.id,
send_configuration: sendConfiguration.id,
status,
response,
response_id: responseId,
updated: now
});
}
}
module.exports = CampaignSender;

View file

@ -172,8 +172,8 @@ async function listTestUsersDTAjax(context, campaignId, params) {
});
}
async function rawGetByIdTx(tx, id) {
const entity = await tx('campaigns').where('campaigns.id', id)
async function rawGetByTx(tx, key, id) {
const entity = await tx('campaigns').where('campaigns.' + key, id)
.leftJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
.groupBy('campaigns.id')
.select([
@ -207,7 +207,7 @@ async function rawGetByIdTx(tx, id) {
async function getByIdTx(tx, context, id, withPermissions = true, content = Content.ALL) {
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'view');
let entity = await rawGetByIdTx(tx, id);
let entity = await rawGetByTx(tx, 'id', id);
if (content === Content.ALL || content === Content.RSS_ENTRY) {
// Return everything
@ -255,22 +255,6 @@ async function getById(context, id, withPermissions = true, content = Content.AL
});
}
async function getByCidTx(tx, context, cid) {
const entity = await tx('campaigns').where('cid', cid).first();
if (!entity) {
shares.throwPermissionDenied();
}
await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.id, 'view');
return entity;
}
async function getByCid(context, cid) {
return await knex.transaction(async tx => {
return getByCidTx(tx, context, cid);
});
}
async function _validateAndPreprocess(tx, context, entity, isCreate, content) {
if (content === Content.ALL || content === Content.WITHOUT_SOURCE_CUSTOM || content === Content.RSS_ENTRY) {
await namespaceHelpers.validateEntity(tx, entity);
@ -405,7 +389,7 @@ async function updateWithConsistencyCheck(context, entity, content) {
await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.id, 'edit');
const existing = await rawGetByIdTx(tx, entity.id);
const existing = await rawGetByTx(tx, 'id', entity.id);
const existingHash = hash(existing, content);
if (existingHash !== entity.originalHash) {
@ -745,4 +729,6 @@ module.exports.getSubscribersQueryGeneratorTx = getSubscribersQueryGeneratorTx;
module.exports.start = start;
module.exports.stop = stop;
module.exports.reset = reset;
module.exports.reset = reset;
module.exports.rawGetBy = rawGetBy;

31
routes/archive.js Normal file
View file

@ -0,0 +1,31 @@
'use strict';
const router = require('../lib/router-async').create();
const CampaignSender = require('../lib/campaign-sender');
router.get('/:campaign/:list/:subscription', (req, res, next) => {
const cs = new CampaignSender();
cs.init({campaignCid: req.params.campaign})
.then(() => cs.getMessage(req.params.list, req.params.subscription))
.then(result => {
const {html} = result;
res.render('partials/tracking-scripts', {
layout: 'archive/layout-raw'
}, (err, scripts) => {
if (err) {
return next(err);
}
html = scripts ? html.replace(/<\/body\b/i, match => scripts + match) : html;
res.render('archive/view-raw', {
layout: 'archive/layout-raw',
message: html
});
});
})
.catch(err => next(err));
});
module.exports = router;

View file

@ -4,6 +4,7 @@ const log = require('npmlog');
const config = require('config');
const router = require('../lib/router-async').create();
const links = require('../models/links');
const interoperableErrors = require('../shared/interoperable-errors');
const trackImg = new Buffer('R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', 'base64');
@ -20,17 +21,6 @@ router.getAsync('/:campaign/:list/:subscription', async (req, res) => {
router.getAsync('/:campaign/:list/:subscription/:link', async (req, res) => {
const notFound = () => {
res.status(404);
return res.render('archive/view', {
layout: 'archive/layout',
message: _('Oops, we couldn\'t find a link for the URL you clicked'),
campaign: {
subject: 'Error 404'
}
});
};
const link = await links.resolve(req.params.link);
if (link) {
@ -40,7 +30,7 @@ router.getAsync('/:campaign/:list/:subscription/:link', async (req, res) => {
return res.redirect(url);
} else {
log.error('Redirect', 'Unresolved URL: <%s>', req.url);
return notFound();
throw new interoperableErrors.NotFoundError('Oops, we couldn\'t find a link for the URL you clicked');
}
});

View file

@ -3,245 +3,11 @@
const config = require('config');
const log = require('npmlog');
const mailers = require('../lib/mailers');
const knex = require('../lib/knex');
const subscriptions = require('../models/subscriptions');
const contextHelpers = require('../lib/context-helpers');
const campaigns = require('../models/campaigns');
const templates = require('../models/templates');
const lists = require('../models/lists');
const fields = require('../models/fields');
const sendConfigurations = require('../models/send-configurations');
const links = require('../models/links');
const {CampaignSource} = require('../shared/campaigns');
const {SubscriptionStatus} = require('../shared/lists');
const tools = require('../lib/tools');
const request = require('request-promise');
const files = require('../models/files');
const htmlToText = require('html-to-text');
const {getPublicUrl} = require('../lib/urls');
const blacklist = require('../models/blacklist');
let libmime = require('libmime');
const CampaignSender = require('../lib/campaign-sender');
const workerId = Number.parseInt(process.argv[2]);
let running = false;
class CampaignSender {
constructor(campaignId) {
this.campaignId = campaignId;
}
async init() {
this.lists = Map(); // listId -> list
this.listsFieldsGrouped = Map(); // listId -> fieldsGrouped
this.attachments = [];
await knex.transaction(async tx => {
this.campaign = await campaigns.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaignId, false, campaigns.Content.ALL)
this.sendConfiguration = await sendConfigurations.getByIdTx(tx, contextHelpers.getAdminContext(), campaign.send_configuration);
for (const listSpec of campaign.lists) {
this.lists.set(listSpec.list) = await lists.getByIdTx(tx, contextHelpers.getAdminContext(), listSpec.list);
this.listsFieldsGrouped.set(listSpec.list) = await fields.listGroupedTx(tx, listSpec.list);
}
if (campaign.source === CampaignSource.TEMPLATE) {
this.template = templates.getByIdTx(tx, contextHelpers.getAdminContext(), this.campaign.data.sourceTemplate, false);
}
const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', this.campaignId);
for (const attachment of attachments) {
this.attachments.push({
filename: attachment.originalname,
path: files.getFilePath('campaign', 'attachment', this.campaignId, attachment.filename)
});
}
});
this.useVerp = config.verp.enabled && sendConfiguration.verp_hostname;
this.useVerpSenderHeader = useVerp && config.verp.disablesenderheader !== true;
}
async sendMessage(listId, email) {
if (await blacklist.isBlacklisted(email)) {
return;
}
const subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, email);
const flds = this.listsFieldsGrouped.get(listId);
const campaign = this.campaign;
const list = this.lists.get(listId);
const sendConfiguration = this.sendConfiguration;
const encryptionKeys = [];
const mergeTags = fields.forHbsWithFieldsGrouped(flds, subscriptionGrouped);
for (const fld of flds) {
if (fld.type === 'gpg' && mergeTags[fld.key]) {
encryptionKeys.push(mergeTags[fld.key].trim());
}
}
let html = '';
let text = '';
let renderTags = false;
if (campaign.source === CampaignSource.URL) {
const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
for (const key in mergeTags) {
form[key] = mergeTags[key];
}
const response = await request.post({
uri: campaign.sourceUrl,
form,
resolveWithFullResponse: true
});
if (response.statusCode !== 200) {
throw new Error(`Received status code ${httpResponse.statusCode} from ${campaign.sourceUrl}`);
}
html = response.body;
text = '';
renderTags = false;
} else if (campaign.source === CampaignSource.CUSTOM || campaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN || campaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
html = campaign.data.sourceCustom.html;
text = campaign.data.sourceCustom.text;
renderTags = true;
} else if (campaign.source === CampaignSource.TEMPLATE) {
const template = this.template;
html = template.html;
text = template.text;
renderTags = true;
}
html = await links.updateLinks(campaign, list, subscriptionGrouped, mergeTags, html);
const attachments = this.attachments.slice();
// replace data: images with embedded attachments
html = html.replace(/(<img\b[^>]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => {
let cid = shortid.generate() + '-attachments@' + campaign.address.split('@').pop();
attachments.push({
path: dataUri,
cid
});
return prefix + 'cid:' + cid;
});
const campaignAddress = [campaign.cid, list.cid, subscriptionGrouped.cid].join('.');
const renderedHtml = renderTags ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, html, false, true) : html;
const renderedText = (text || '').trim()
? (renderTags ? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, text) : text)
: htmlToText.fromString(renderedHtml, {wordwrap: 130});
let listUnsubscribe = null;
if (!list.listunsubscribe_disabled) {
listUnsubscribe = campaign.unsubscribe_url
? tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, campaign.unsubscribe_url)
: getPublicUrl('/subscription/' + list.cid + '/unsubscribe/' + subscriptionGrouped.subscription.cid);
}
const mailer = await mailers.getOrCreateMailer(sendConfiguration.id);
await mailer.throttleWait();
const getOverridable = key => {
if (sendConfiguration[key + '_overridable'] && this.campaign[key + '_override'] !== null) {
return campaign[key + '_override'];
} else {
return sendConfiguration[key];
}
}
const mail = {
from: {
name: getOverridable('from_name'),
address: getOverridable('from_email')
},
replyTo: getOverridable('reply_to'),
xMailer: sendConfiguration.x_mailer ? sendConfiguration.x_mailer : false,
to: {
name: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, list.to_name, false, false),
address: subscriptionGrouped.email
},
sender: this.useVerpSenderHeader ? campaignAddress + '@' + sendConfiguration.verp_hostname : false,
envelope: this.useVerp ? {
from: campaignAddress + '@' + sendConfiguration.verp_hostname,
to: subscriptionGrouped.email
} : false,
headers: {
'x-fbl': campaignAddress,
// custom header for SparkPost
'x-msys-api': JSON.stringify({
campaign_id: campaignAddress
}),
// custom header for SendGrid
'x-smtpapi': JSON.stringify({
unique_args: {
campaign_id: campaignAddress
}
}),
// custom header for Mailgun
'x-mailgun-variables': JSON.stringify({
campaign_id: campaignAddress
}),
'List-ID': {
prepared: true,
value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + getPublicUrl() + '>'
}
},
list: {
unsubscribe: listUnsubscribe
},
subject: tools.formatMessage(campaign, list, subscriptionGrouped, mergeTags, getOverridable('subject'), false, false),
html: renderedHtml,
text: renderedText,
attachments,
encryptionKeys
};
let status;
let response;
try {
const info = await mailer.sendMassMail(mail);
status = SubscriptionStatus.SUBSCRIBED;
response = info.response || info.messageId;
await knex('campaigns').where('id', campaign.id).increment('delivered');
} catch (err) {
status = SubscriptionStatus.BOUNCED;
response = err.response || err.message;
await knex('campaigns').where('id', campaign.id).increment('delivered').increment('bounced');
}
const responseId = response.split(/\s+/).pop();
const now = new Date();
await knex('campaign_messages').insert({
campaign: this.campaignId,
list: listId,
subscriptions: subscriptionGrouped.id,
send_configuration: sendConfiguration.id,
status,
response,
response_id: responseId,
updated: now
});
log.verbose('Senders', 'Message sent and status updated for %s', subscriptionGrouped.cid);
}
}
async function processMessages(campaignId, subscribers) {
if (running) {
log.error('Senders', `Worker ${workerId} assigned work while working`);
@ -250,12 +16,13 @@ async function processMessages(campaignId, subscribers) {
running = true;
const cs = new CampaignSender(campaignId);
await cs.init()
const cs = new CampaignSender();
await cs.init({campaignId})
for (const subData of subscribers) {
try {
await cs.sendMessage(subData.listId, subData.email);
log.verbose('Senders', 'Message sent and status updated for %s:%s', subData.listId, subData.email);
} catch (err) {
log.error('Senders', `Sending message to ${subData.listId}:${subData.email} failed with error: ${err.message}`)
}

View file

@ -0,0 +1 @@
{{{body}}}

View file

@ -0,0 +1 @@
{{{message}}}