From a24aba8c4d6f64c7e82e0dda23ebd7d0df5c6342 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 26 Apr 2016 17:51:29 +0300 Subject: [PATCH 1/4] Added bounce instruction links --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8a04c3fc..e8380c46 100644 --- a/README.md +++ b/README.md @@ -80,9 +80,9 @@ In the IDE, start Mailtrain via `Run > Start Mailtrain` and access your site via Mailtrain uses webhooks integration to detect bounces and spam complaints. Currently supported webhooks are: * **AWS SES** – create a SNS topic for complaints and bounces and use `http://domain/webhooks/aws` as the subscriber URL for these topics - * **SparkPost** – use `http://domain/webhooks/sparkpost` as the webhook URL for bounces and complaints - * **SendGrid** – use `http://domain/webhooks/sendgrid` as the webhook URL for bounces and complaints - * **Mailgun** – use `http://domain/webhooks/mailgun` as the webhook URL for bounces and complaints + * **SparkPost** – use `http://domain/webhooks/sparkpost` as the webhook URL for bounces and complaints ([instructions](https://github.com/andris9/mailtrain/wiki/Setting-up-Webhooks-for-SparkPost)) + * **SendGrid** – use `http://domain/webhooks/sendgrid` as the webhook URL for bounces and complaints ([instructions](https://github.com/andris9/mailtrain/wiki/Setting-up-Webhooks-for-SendGrid)) + * **Mailgun** – use `http://domain/webhooks/mailgun` as the webhook URL for bounces and complaints ([instructions](https://github.com/andris9/mailtrain/wiki/Setting-up-Webhooks-for-Mailgun)) Additionally Mailtrain (v1.1+) is able to use VERP-based bounce handling. This would require to have a compatible SMTP relay (the services mentioned above strip out or block VERP addresses in the SMTP envelope) and you also need to set up special MX DNS name that points to your Mailtrain installation server. From e4c71f4026e9f20aad87ee2ab9dcde5a4b330049 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 26 Apr 2016 19:07:07 +0300 Subject: [PATCH 2/4] First take on the "send from url" feature --- README.md | 1 + lib/mailer.js | 5 ++ lib/models/campaigns.js | 2 +- meta.json | 2 +- package.json | 1 + services/sender.js | 141 +++++++++++++++++++++--------------- setup/sql/upgrade-00004.sql | 12 +++ views/campaigns/create.hbs | 32 +++++--- views/campaigns/edit.hbs | 111 +++++++++++++++------------- 9 files changed, 190 insertions(+), 117 deletions(-) create mode 100644 setup/sql/upgrade-00004.sql diff --git a/README.md b/README.md index e8380c46..738262d1 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Subscribe to Mailtrain Newsletter [here](http://mailtrain.org/subscription/EysIv ## Upgrade * Replace old files with new ones by running in the Mailtrain folder `git pull origin master` + * Run `npm install --production` in the Mailtrain folder ## Using environment variables diff --git a/lib/mailer.js b/lib/mailer.js index 763ed6ed..9ac3f861 100644 --- a/lib/mailer.js +++ b/lib/mailer.js @@ -9,6 +9,7 @@ let Handlebars = require('handlebars'); let fs = require('fs'); let path = require('path'); let templates = new Map(); +let htmlToText = require('html-to-text'); module.exports.transport = false; @@ -54,6 +55,10 @@ module.exports.sendMail = (mail, template, callback) => { if (textRenderer) { mail.text = textRenderer(template.data || {}); + } else if (mail.html) { + mail.text = htmlToText.fromString(mail.html, { + wordwrap: 130 + }); } module.exports.transport.sendMail(mail, callback); diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index 84c48389..672541df 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -8,7 +8,7 @@ let segments = require('./segments'); let subscriptions = require('./subscriptions'); let shortid = require('shortid'); -let allowedKeys = ['description', 'from', 'address', 'subject', 'template', 'list', 'segment', 'html', 'text']; +let allowedKeys = ['description', 'from', 'address', 'subject', 'template', 'template_url', 'list', 'segment', 'html', 'text']; module.exports.list = (start, limit, callback) => { db.getConnection((err, connection) => { diff --git a/meta.json b/meta.json index 18fc9078..b4a9b4a7 100644 --- a/meta.json +++ b/meta.json @@ -1,3 +1,3 @@ { - "schemaVersion": 3 + "schemaVersion": 4 } diff --git a/package.json b/package.json index 7272b941..52d7fcd8 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "geoip-ultralight": "^0.1.3", "handlebars": "^4.0.5", "hbs": "^4.0.0", + "html-to-text": "^2.1.0", "humanize": "0.0.9", "isemail": "^2.1.0", "morgan": "^1.7.0", diff --git a/services/sender.js b/services/sender.js index ff1202bb..02f8cd08 100644 --- a/services/sender.js +++ b/services/sender.js @@ -13,6 +13,8 @@ let settings = require('../lib/models/settings'); let links = require('../lib/models/links'); let shortid = require('shortid'); let url = require('url'); +let htmlToText = require('html-to-text'); +let request = require('request'); function findUnsent(callback) { db.getConnection((err, connection) => { @@ -149,73 +151,98 @@ function formatMessage(message, callback) { } }); - links.updateLinks(campaign, list, message.subscription, configItems.serviceUrl, campaign.html, (err, html) => { - if (err) { - return callback(err); - } + let renderAndSend = (html, text, renderTags) => { + links.updateLinks(campaign, list, message.subscription, configItems.serviceUrl, html, (err, html) => { + if (err) { + return callback(err); + } - // replace data: images with embedded attachments - let attachments = []; - html = html.replace(/(]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => { - let cid = shortid.generate() + '-attachments@' + campaign.address.split('@').pop(); - attachments.push({ - path: dataUri, - cid + // replace data: images with embedded attachments + let attachments = []; + html = html.replace(/(]* src\s*=[\s"']*)(data:[^"'>\s]+)/gi, (match, prefix, dataUri) => { + let cid = shortid.generate() + '-attachments@' + campaign.address.split('@').pop(); + attachments.push({ + path: dataUri, + cid + }); + return prefix + 'cid:' + cid; }); - return prefix + 'cid:' + cid; - }); - let campaignAddress = [campaign.cid, list.cid, message.subscription.cid].join('.'); + let campaignAddress = [campaign.cid, list.cid, message.subscription.cid].join('.'); - return callback(null, { - from: { - name: campaign.from, - address: campaign.address - }, - xMailer: 'Mailtrain Mailer (+https://mailtrain.org)', - to: { - name: [].concat(message.subscription.firstName || []).concat(message.subscription.lastName || []).join(' '), - address: message.subscription.email - }, - sender: useVerp ? campaignAddress + '@' + configItems.verpHostname : false, + let renderedHtml = renderTags ? tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, html) : html; - envelope: useVerp ? { - from: campaignAddress + '@' + configItems.verpHostname, - to: message.subscription.email - } : false, + let renderedText = (text || '').trim() ? (renderTags ? tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, text) : text) : htmlToText.fromString(renderedHtml, { + wordwrap: 130 + }); - headers: { - 'x-fbl': campaignAddress, - // custom header for SparkPost - 'x-msys-api': JSON.stringify({ - campaign_id: campaignAddress - }), - // custom header for SendGrid - 'x-smtpapi': JSON.stringify({ - unique_args: { + return callback(null, { + from: { + name: campaign.from, + address: campaign.address + }, + xMailer: 'Mailtrain Mailer (+https://mailtrain.org)', + to: { + name: [].concat(message.subscription.firstName || []).concat(message.subscription.lastName || []).join(' '), + address: message.subscription.email + }, + sender: useVerp ? campaignAddress + '@' + configItems.verpHostname : false, + + envelope: useVerp ? { + from: campaignAddress + '@' + configItems.verpHostname, + to: message.subscription.email + } : false, + + headers: { + 'x-fbl': campaignAddress, + // custom header for SparkPost + 'x-msys-api': JSON.stringify({ campaign_id: campaignAddress + }), + // custom header for SendGrid + 'x-smtpapi': JSON.stringify({ + unique_args: { + campaign_id: campaignAddress + } + }), + // custom header for Mailgun + 'x-mailgun-variables': JSON.stringify({ + campaign_id: campaignAddress + }), + 'List-ID': { + prepared: true, + value: '"' + list.name.replace(/[^a-z0-9\s'.,\-]/g, '').trim() + '" <' + list.cid + '.' + (url.parse(configItems.serviceUrl).hostname || 'localhost') + '>' } - }), - // custom header for Mailgun - 'x-mailgun-variables': JSON.stringify({ - campaign_id: campaignAddress - }), - 'List-ID': { - prepared: true, - value: '"' + list.name.replace(/[^a-z0-9\s'.,\-]/g, '').trim() + '" <' + list.cid + '.' + (url.parse(configItems.serviceUrl).hostname || 'localhost') + '>' - } - }, - list: { - unsubscribe: url.resolve(configItems.serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + message.subscription.cid + '?auto=yes') - }, - subject: tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, campaign.subject), - html: tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, html), - text: tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, campaign.text), + }, + list: { + unsubscribe: url.resolve(configItems.serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + message.subscription.cid + '?auto=yes') + }, + subject: tools.formatMessage(configItems.serviceUrl, campaign, list, message.subscription, campaign.subject), + html: renderedHtml, + text: renderedText, - attachments, - encryptionKeys + attachments, + encryptionKeys + }); }); - }); + }; + + if (!campaign.template && campaign.templateUrl) { + request.post({ + url: campaign.templateUrl, + form: message.subscription.mergeTags + }, (err, httpResponse, body) => { + if (err) { + return callback(err); + } + if (httpResponse.statusCode !== 200) { + return callback(new Error('Received status code ' + httpResponse.statusCode + ' from ' + campaign.templateUrl)); + } + renderAndSend(body && body.toString(), '', false); + }); + } else { + renderAndSend(campaign.html, campaign.text, true); + } }); }); }); diff --git a/setup/sql/upgrade-00004.sql b/setup/sql/upgrade-00004.sql new file mode 100644 index 00000000..bf2c2ef6 --- /dev/null +++ b/setup/sql/upgrade-00004.sql @@ -0,0 +1,12 @@ +# Header section +# Define incrementing schema version number +SET @schema_version = '4'; + +# Adds new column 'template_url' to campaigns table +# Indicates that this campaign should fetch message content from this URL +ALTER TABLE `campaigns` ADD COLUMN `template_url` varchar(255) CHARACTER SET ascii DEFAULT NULL AFTER `template`; + +# Footer section +LOCK TABLES `settings` WRITE; +INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version; +UNLOCK TABLES; diff --git a/views/campaigns/create.hbs b/views/campaigns/create.hbs index 3cc86363..8b238f31 100644 --- a/views/campaigns/create.hbs +++ b/views/campaigns/create.hbs @@ -53,15 +53,29 @@
- - Not required. Creates a campaign specific copy from a template that you can later edit + +

+ Select a template: +

+
+ + Selecting a template creates a campaign specific copy from it +
+

+ Or alternatively use an URL as the message content source: +

+
+ + If a message is sent then this URL will be POSTed to using Merge Tags as POST body. Use this if you want to generate the HTML message yourself +
+
diff --git a/views/campaigns/edit.hbs b/views/campaigns/edit.hbs index 019fa9b1..e33bcb73 100644 --- a/views/campaigns/edit.hbs +++ b/views/campaigns/edit.hbs @@ -106,60 +106,73 @@ Template Settings -
-
- -
-

- Merge tags are tags that are replaced before sending out the message. The format of the merge tag is the following: [TAG_NAME] or [TAG_NAME/fallback] where fallback is an optional - text value used when TAG_NAME is empty. -

- -
    -
  • - [FIRST_NAME] – first name of the subcriber -
    • -
  • - [LAST_NAME] – last name of the subcriber -
    • -
  • - [FULL_NAME] – first and last names of the subcriber joined -
    • -
  • - [LINK_UNSUBSCRIBE] – URL that points to the preferences page of the subscriber -
    • -
  • - [LINK_PREFERENCES] – URL that points to the unsubscribe page -
    • -
  • - [LINK_BROWSER] – URL to preview the message in a browser -
    • -
-

- In addition to that any custom field can have its own merge tag. -

+ {{#if templateUrl}} +
+ +
+ + If a message is sent then this URL will be POSTed to using Merge Tags as POST body. Use this if you want to generate the HTML message yourself
-
+ {{else}} -
- -
- {{#if disableWysiwyg}} -
{{html}}
- - {{else}} - - {{/if}} -
-
-
- -
- +
+
+ +
+

+ Merge tags are tags that are replaced before sending out the message. The format of the merge tag is the following: [TAG_NAME] or [TAG_NAME/fallback] where fallback is an optional + text value used when TAG_NAME is empty. +

+ +
    +
  • + [FIRST_NAME] – first name of the subcriber +
    • +
  • + [LAST_NAME] – last name of the subcriber +
    • +
  • + [FULL_NAME] – first and last names of the subcriber joined +
    • +
  • + [LINK_UNSUBSCRIBE] – URL that points to the preferences page of the subscriber +
    • +
  • + [LINK_PREFERENCES] – URL that points to the unsubscribe page +
    • +
  • + [LINK_BROWSER] – URL to preview the message in a browser +
    • +
+

+ In addition to that any custom field can have its own merge tag. +

+
+
-
+ +
+ +
+ {{#if disableWysiwyg}} +
{{html}}
+ + {{else}} + + {{/if}} +
+
+ +
+ +
+ +
+
+ + {{/if}}
From 43b1f1e319617ef2f65a8e87caa1b1fa47eb3304 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 26 Apr 2016 20:29:57 +0300 Subject: [PATCH 3/4] Include campaign links in form fields --- lib/tools.js | 23 +++++++++++++++-------- services/sender.js | 6 +++++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/tools.js b/lib/tools.js index 79cde0a1..6a8a98c8 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -15,7 +15,8 @@ module.exports = { createSlug, updateMenu, validateEmail, - formatMessage + formatMessage, + getMessageLinks }; function toDbKey(key) { @@ -147,17 +148,23 @@ function validateEmail(address, checkBlocked, callback) { }); } +function getMessageLinks(serviceUrl, campaign, list, subscription) { + return { + LINK_UNSUBSCRIBE: urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + subscription.cid + '?auto=yes&c=' + campaign.cid), + LINK_PREFERENCES: urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/manage/' + subscription.cid), + LINK_BROWSER: urllib.resolve(serviceUrl, '/archive/' + campaign.cid + '/' + list.cid + '/' + subscription.cid) + }; +} + function formatMessage(serviceUrl, campaign, list, subscription, message, filter) { filter = typeof filter === 'function' ? filter : (str => str); + let links = getMessageLinks(serviceUrl, campaign, list, subscription); + let getValue = key => { - switch ((key || '').toString().toUpperCase().trim()) { - case 'LINK_UNSUBSCRIBE': - return urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + subscription.cid + '?auto=yes&c=' + campaign.cid); - case 'LINK_PREFERENCES': - return urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/manage/' + subscription.cid); - case 'LINK_BROWSER': - return urllib.resolve(serviceUrl, '/archive/' + campaign.cid + '/' + list.cid + '/' + subscription.cid); + key = (key || '').toString().toUpperCase().trim(); + if (links.hasOwnProperty(key)) { + return links[key]; } if (subscription.mergeTags.hasOwnProperty(key)) { return subscription.mergeTags[key]; diff --git a/services/sender.js b/services/sender.js index 02f8cd08..314998b7 100644 --- a/services/sender.js +++ b/services/sender.js @@ -228,9 +228,13 @@ function formatMessage(message, callback) { }; if (!campaign.template && campaign.templateUrl) { + let form = tools.getMessageLinks(configItems.serviceUrl, campaign, list, message.subscription); + Object.keys(message.subscription.mergeTags).forEach(key => { + form[key] = message.subscription.mergeTags[key]; + }); request.post({ url: campaign.templateUrl, - form: message.subscription.mergeTags + form }, (err, httpResponse, body) => { if (err) { return callback(err); From ffcc873ef3ec38c8798ae8f988bdbfd5424fdbc8 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 26 Apr 2016 21:14:48 +0300 Subject: [PATCH 4/4] Support URL-based archive pages --- routes/archive.js | 51 ++++++++++++++++++++++++++++++++++------------ services/sender.js | 2 +- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/routes/archive.js b/routes/archive.js index 8e08e844..8ae7348f 100644 --- a/routes/archive.js +++ b/routes/archive.js @@ -7,6 +7,7 @@ let lists = require('../lib/models/lists'); let subscriptions = require('../lib/models/subscriptions'); let tools = require('../lib/tools'); let express = require('express'); +let request = require('request'); let router = new express.Router(); router.get('/:campaign/:list/:subscription', (req, res, next) => { @@ -64,21 +65,45 @@ router.get('/:campaign/:list/:subscription', (req, res, next) => { return next(err); } - // rewrite links to count clicks - links.updateLinks(campaign, list, subscription, serviceUrl, campaign.html, (err, html) => { - if (err) { - req.flash('danger', err.message || err); - return res.redirect('/'); - } + let renderAndShow = (html, renderTags) => { - res.render('archive/view', { - layout: 'archive/layout', - message: tools.formatMessage(serviceUrl, campaign, list, subscription, html), - campaign, - list, - subscription + // rewrite links to count clicks + links.updateLinks(campaign, list, subscription, serviceUrl, html, (err, html) => { + if (err) { + req.flash('danger', err.message || err); + return res.redirect('/'); + } + + res.render('archive/view', { + layout: 'archive/layout', + message: renderTags ? tools.formatMessage(serviceUrl, campaign, list, subscription, html) : html, + campaign, + list, + subscription + }); }); - }); + }; + + if (campaign.templateUrl) { + let form = tools.getMessageLinks(serviceUrl, campaign, list, subscription); + Object.keys(subscription.mergeTags).forEach(key => { + form[key] = subscription.mergeTags[key]; + }); + request.post({ + url: campaign.templateUrl, + form + }, (err, httpResponse, body) => { + if (err) { + return next(err); + } + if (httpResponse.statusCode !== 200) { + return next(new Error('Received status code ' + httpResponse.statusCode + ' from ' + campaign.templateUrl)); + } + renderAndShow(body && body.toString(), false); + }); + } else { + renderAndShow(campaign.html, true); + } }); }); }); diff --git a/services/sender.js b/services/sender.js index 314998b7..0dd18a7a 100644 --- a/services/sender.js +++ b/services/sender.js @@ -227,7 +227,7 @@ function formatMessage(message, callback) { }); }; - if (!campaign.template && campaign.templateUrl) { + if (campaign.templateUrl) { let form = tools.getMessageLinks(configItems.serviceUrl, campaign, list, message.subscription); Object.keys(message.subscription.mergeTags).forEach(key => { form[key] = message.subscription.mergeTags[key];