From 35bce325291efeeb46333bee5398163e4dfce473 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Fri, 9 Sep 2016 23:09:04 +0300 Subject: [PATCH] Full support for message attachments --- lib/models/campaigns.js | 11 ++- routes/archive.js | 117 ++++++++++++++++++---------- routes/campaigns.js | 2 +- services/sender.js | 166 +++++++++++++++++++++++++++------------- setup/sql/mailtrain.sql | 20 ++++- views/archive/view.hbs | 19 ++++- 6 files changed, 229 insertions(+), 106 deletions(-) diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index f7e4fbec..c8e81c94 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -374,7 +374,10 @@ module.exports.getAttachments = (campaign, callback) => { if (err) { return callback(err); } - connection.query('SELECT `id`, `filename`, `size`, `created` FROM `attachments` WHERE `campaign`=?', [campaign], (err, rows) => { + + let keys = ['id', 'filename', 'content_type', 'size', 'created']; + + connection.query('SELECT `' + keys.join('`, `') + '` FROM `attachments` WHERE `campaign`=?', [campaign], (err, rows) => { connection.release(); if (err) { return callback(err); @@ -455,14 +458,14 @@ module.exports.deleteAttachment = (id, attachment, callback) => { }); }; -module.exports.getAttachment = (id, attachment, callback) => { +module.exports.getAttachment = (campaign, attachment, callback) => { db.getConnection((err, connection) => { if (err) { return callback(err); } - let query = 'SELECT `filename`, `content_type`, `content` FROM `attachments` WHERE `id`=? AND `campaign`=? LIMIT 1'; - connection.query(query, [attachment, id], (err, rows) => { + let query = 'SELECT * FROM `attachments` WHERE `id`=? AND `campaign`=? LIMIT 1'; + connection.query(query, [attachment, campaign], (err, rows) => { connection.release(); if (err) { return callback(err); diff --git a/routes/archive.js b/routes/archive.js index 02e54408..8cd8245b 100644 --- a/routes/archive.js +++ b/routes/archive.js @@ -9,8 +9,9 @@ let tools = require('../lib/tools'); let express = require('express'); let request = require('request'); let router = new express.Router(); +let passport = require('../lib/passport'); -router.get('/:campaign/:list/:subscription', (req, res, next) => { +router.get('/:campaign/:list/:subscription', passport.csrfProtection, (req, res, next) => { settings.get('serviceUrl', (err, serviceUrl) => { if (err) { req.flash('danger', err.message || err); @@ -53,54 +54,86 @@ router.get('/:campaign/:list/:subscription', (req, res, next) => { return next(err); } - let renderHtml = (html, renderTags) => { - res.render('archive/view', { - layout: 'archive/layout', - message: renderTags ? tools.formatMessage(serviceUrl, campaign, list, subscription, html) : html, - campaign, - list, - subscription - }); - }; - - let renderAndShow = (html, renderTags) => { - if (req.query.track === 'no') { - return renderHtml(html, renderTags); + campaigns.getAttachments(campaign.id, (err, attachments) => { + if (err) { + req.flash('danger', err.message || err); + return res.redirect('/'); } - // rewrite links to count clicks - links.updateLinks(campaign, list, subscription, serviceUrl, html, (err, html) => { - if (err) { - req.flash('danger', err.message || err); - return res.redirect('/'); - } - renderHtml(html, renderTags); - }); - }; - if (campaign.sourceUrl) { - let form = tools.getMessageLinks(serviceUrl, campaign, list, subscription); - Object.keys(subscription.mergeTags).forEach(key => { - form[key] = subscription.mergeTags[key]; - }); - request.post({ - url: campaign.sourceUrl, - form - }, (err, httpResponse, body) => { - if (err) { - return next(err); + let renderHtml = (html, renderTags) => { + res.render('archive/view', { + layout: 'archive/layout', + message: renderTags ? tools.formatMessage(serviceUrl, campaign, list, subscription, html) : html, + campaign, + list, + subscription, + attachments, + csrfToken: req.csrfToken() + }); + }; + + let renderAndShow = (html, renderTags) => { + if (req.query.track === 'no') { + return renderHtml(html, renderTags); } - if (httpResponse.statusCode !== 200) { - return next(new Error('Received status code ' + httpResponse.statusCode + ' from ' + campaign.sourceUrl)); - } - renderAndShow(body && body.toString(), false); - }); - } else { - renderAndShow(campaign.html, true); - } + // rewrite links to count clicks + links.updateLinks(campaign, list, subscription, serviceUrl, html, (err, html) => { + if (err) { + req.flash('danger', err.message || err); + return res.redirect('/'); + } + renderHtml(html, renderTags); + }); + }; + + if (campaign.sourceUrl) { + let form = tools.getMessageLinks(serviceUrl, campaign, list, subscription); + Object.keys(subscription.mergeTags).forEach(key => { + form[key] = subscription.mergeTags[key]; + }); + request.post({ + url: campaign.sourceUrl, + form + }, (err, httpResponse, body) => { + if (err) { + return next(err); + } + if (httpResponse.statusCode !== 200) { + return next(new Error('Received status code ' + httpResponse.statusCode + ' from ' + campaign.sourceUrl)); + } + renderAndShow(body && body.toString(), false); + }); + } else { + renderAndShow(campaign.html, true); + } + }); }); }); }); }); }); +router.post('/attachment/download', passport.parseForm, passport.csrfProtection, (req, res) => { + let url = '/archive/' + encodeURIComponent(req.body.campaign || '') + '/' + encodeURIComponent(req.body.list || '') + '/' + encodeURIComponent(req.body.subscription || ''); + campaigns.getByCid(req.body.campaign, (err, campaign) => { + if (err || !campaign) { + req.flash('danger', err && err.message || err || 'Could not find campaign with specified ID'); + return res.redirect(url); + } + campaigns.getAttachment(campaign.id, Number(req.body.attachment), (err, attachment) => { + if (err) { + req.flash('danger', err && err.message || err); + return res.redirect(url); + } else if (!attachment) { + req.flash('warning', 'Attachment not found'); + return res.redirect(url); + } + + res.set('Content-Disposition', 'attachment; filename="' + encodeURIComponent(attachment.filename).replace(/['()]/g, escape) + '"'); + res.set('Content-Type', attachment.contentType); + res.send(attachment.content); + }); + }); +}); + module.exports = router; diff --git a/routes/campaigns.js b/routes/campaigns.js index 8b6ed325..f41143a4 100644 --- a/routes/campaigns.js +++ b/routes/campaigns.js @@ -828,7 +828,7 @@ router.post('/attachment/download', passport.parseForm, passport.csrfProtection, req.flash('danger', err && err.message || err); return res.redirect('/campaigns/edit/' + campaign.id + '?tab=attachments'); } else if (!attachment) { - req.flash('success', 'Attachment uploaded'); + req.flash('warning', 'Attachment not found'); return res.redirect('/campaigns/edit/' + campaign.id + '?tab=attachments'); } diff --git a/services/sender.js b/services/sender.js index 87ddfe20..6b15ebeb 100644 --- a/services/sender.js +++ b/services/sender.js @@ -18,6 +18,9 @@ let request = require('request'); let caches = require('../lib/caches'); let libmime = require('libmime'); +let attachmentCache = new Map(); +let attachmentCacheSize = 0; + function findUnsent(callback) { let returnUnsent = (row, campaign) => { db.getConnection((err, connection) => { @@ -195,6 +198,54 @@ function findUnsent(callback) { }); } +function getAttachments(campaign, callback) { + campaigns.getAttachments(campaign.id, (err, attachments) => { + if (err) { + return callback(err); + } + if (!attachments) { + return callback(null, []); + } + + let response = []; + let pos = 0; + let getNextAttachment = () => { + if (pos >= attachments.length) { + return callback(null, response); + } + let attachment = attachments[pos++]; + let aid = campaign.id + ':' + attachment.id; + if (attachmentCache.has(aid)) { + response.push(attachmentCache.get(aid)); + return setImmediate(getNextAttachment); + } + campaigns.getAttachment(campaign.id, attachment.id, (err, attachment) => { + if (err) { + return callback(err); + } + if (!attachment || !attachment.content) { + return setImmediate(getNextAttachment); + } + + response.push(attachment); + + // make sure we do not cache more buffers than 30MB + if (attachmentCacheSize + attachment.content.length > 30 * 1024 * 1024) { + attachmentCacheSize = 0; + attachmentCache.clear(); + } + + attachmentCache.set(aid, attachment); + attachmentCacheSize += attachment.content.length; + + return setImmediate(getNextAttachment); + }); + }; + + getNextAttachment(); + }); +} + function formatMessage(message, callback) { campaigns.get(message.campaignId, false, (err, campaign) => { if (err) { @@ -255,71 +306,76 @@ function formatMessage(message, callback) { } // replace data: images with embedded attachments - let attachments = []; - html = html.replace(/(]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => { - let cid = shortid.generate() + '-attachments@' + campaign.address.split('@').pop(); - attachments.push({ - path: dataUri, - cid + getAttachments(campaign, (err, attachments) => { + if (err) { + return callback(err); + } + + html = html.replace(/(]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => { + let cid = shortid.generate() + '-attachments@' + campaign.address.split('@').pop(); + attachments.push({ + path: dataUri, + cid + }); + return prefix + 'cid:' + cid; }); - return prefix + 'cid:' + cid; - }); - let campaignAddress = [campaign.cid, list.cid, message.subscription.cid].join('.'); + let campaignAddress = [campaign.cid, list.cid, message.subscription.cid].join('.'); - let renderedHtml = renderTags ? tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, html) : html; + let renderedHtml = renderTags ? tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, html) : html; - let renderedText = (text || '').trim() ? (renderTags ? tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, text) : text) : htmlToText.fromString(renderedHtml, { - wordwrap: 130 - }); + let renderedText = (text || '').trim() ? (renderTags ? tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, text) : text) : htmlToText.fromString(renderedHtml, { + wordwrap: 130 + }); - return callback(null, { - from: { - name: campaign.from, - address: campaign.address - }, - xMailer: 'Mailtrain Mailer (+https://mailtrain.org)', - to: { - name: [].concat(message.subscription.firstName || []).concat(message.subscription.lastName || []).join(' '), - address: message.subscription.email - }, - sender: useVerp ? campaignAddress + '@' + configItems.verpHostname : false, + return callback(null, { + from: { + name: campaign.from, + address: campaign.address + }, + xMailer: 'Mailtrain Mailer (+https://mailtrain.org)', + to: { + name: [].concat(message.subscription.firstName || []).concat(message.subscription.lastName || []).join(' '), + address: message.subscription.email + }, + sender: useVerp ? campaignAddress + '@' + configItems.verpHostname : false, - envelope: useVerp ? { - from: campaignAddress + '@' + configItems.verpHostname, - to: message.subscription.email - } : false, + envelope: useVerp ? { + from: campaignAddress + '@' + configItems.verpHostname, + to: message.subscription.email + } : false, - headers: { - 'x-fbl': campaignAddress, - // custom header for SparkPost - 'x-msys-api': JSON.stringify({ - campaign_id: campaignAddress - }), - // custom header for SendGrid - 'x-smtpapi': JSON.stringify({ - unique_args: { + headers: { + 'x-fbl': campaignAddress, + // custom header for SparkPost + 'x-msys-api': JSON.stringify({ campaign_id: campaignAddress + }), + // custom header for SendGrid + 'x-smtpapi': JSON.stringify({ + unique_args: { + campaign_id: campaignAddress + } + }), + // custom header for Mailgun + 'x-mailgun-variables': JSON.stringify({ + campaign_id: campaignAddress + }), + 'List-ID': { + prepared: true, + value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + (url.parse(configItems.serviceUrl).hostname || 'localhost') + '>' } - }), - // custom header for Mailgun - 'x-mailgun-variables': JSON.stringify({ - campaign_id: campaignAddress - }), - 'List-ID': { - prepared: true, - value: libmime.encodeWords(list.name) + ' <' + list.cid + '.' + (url.parse(configItems.serviceUrl).hostname || 'localhost') + '>' - } - }, - list: { - unsubscribe: url.resolve(configItems.serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + message.subscription.cid + '?auto=yes') - }, - subject: tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, campaign.subject), - html: renderedHtml, - text: renderedText, + }, + list: { + unsubscribe: url.resolve(configItems.serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + message.subscription.cid + '?auto=yes') + }, + subject: tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, campaign.subject), + html: renderedHtml, + text: renderedText, - attachments, - encryptionKeys + attachments, + encryptionKeys + }); }); }); }; diff --git a/setup/sql/mailtrain.sql b/setup/sql/mailtrain.sql index 4da0bd48..3ada3fdf 100644 --- a/setup/sql/mailtrain.sql +++ b/setup/sql/mailtrain.sql @@ -1,6 +1,18 @@ SET UNIQUE_CHECKS=0; SET FOREIGN_KEY_CHECKS=0; +CREATE TABLE `attachments` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `campaign` int(11) unsigned NOT NULL, + `filename` varchar(255) CHARACTER SET utf8mb4 NOT NULL DEFAULT '', + `content_type` varchar(100) CHARACTER SET ascii NOT NULL DEFAULT '', + `content` longblob, + `size` int(11) NOT NULL, + `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `campaign` (`campaign`), + CONSTRAINT `attachments_ibfk_1` FOREIGN KEY (`campaign`) REFERENCES `campaigns` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `campaign` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `list` int(11) unsigned NOT NULL, @@ -49,6 +61,7 @@ CREATE TABLE `campaigns` ( `html_prepared` longtext, `text` longtext, `status` tinyint(4) unsigned NOT NULL DEFAULT '1', + `tracking_disabled` tinyint(4) unsigned NOT NULL DEFAULT '0', `scheduled` timestamp NULL DEFAULT NULL, `status_change` timestamp NULL DEFAULT NULL, `delivered` int(11) unsigned NOT NULL DEFAULT '0', @@ -72,6 +85,7 @@ CREATE TABLE `confirmations` ( `cid` varchar(255) CHARACTER SET ascii NOT NULL, `list` int(11) unsigned NOT NULL, `email` varchar(255) NOT NULL, + `opt_in_ip` varchar(100) DEFAULT NULL, `data` text NOT NULL, `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), @@ -186,7 +200,7 @@ CREATE TABLE `segments` ( `created` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `list` (`list`), - KEY `name` (`name`(191)), + KEY `name` (`name`), CONSTRAINT `segments_ibfk_1` FOREIGN KEY (`list`) REFERENCES `lists` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `settings` ( @@ -195,7 +209,7 @@ CREATE TABLE `settings` ( `value` text NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `key` (`key`) -) ENGINE=InnoDB AUTO_INCREMENT=34 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=36 DEFAULT CHARSET=utf8mb4; INSERT INTO `settings` (`id`, `key`, `value`) VALUES (1,'smtp_hostname','localhost'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (2,'smtp_port','465'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (3,'smtp_encryption','TLS'); @@ -212,7 +226,7 @@ INSERT INTO `settings` (`id`, `key`, `value`) VALUES (13,'default_from','My Awes INSERT INTO `settings` (`id`, `key`, `value`) VALUES (14,'default_address','admin@example.com'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (15,'default_subject','Test message'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (16,'default_homepage','http://localhost:3000/'); -INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','17'); +INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','19'); CREATE TABLE `subscription` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `cid` varchar(255) CHARACTER SET ascii NOT NULL, diff --git a/views/archive/view.hbs b/views/archive/view.hbs index 9c4d116a..10bc3dd5 100644 --- a/views/archive/view.hbs +++ b/views/archive/view.hbs @@ -1,2 +1,19 @@ + {{{message}}} -{{{message}}} +{{#if attachments}} +
    + {{#each attachments}} +
  • + {{size}} +
    + + + + + + +
    +
    • + {{/each}} +
+{{/if}}