From 4f2d66c30c00a541972b37996d4f9d7fe20d2395 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 3 May 2016 19:21:01 +0300 Subject: [PATCH] Use juice to prepare html messages --- lib/mailer.js | 29 ++++++--- lib/models/campaigns.js | 112 +++++++++++++++++++++------------- lib/tools.js | 38 +++++++++++- package.json | 2 + services/sender.js | 2 +- setup/sql/mailtrain.sql | 22 ++++++- setup/sql/upgrade-00008.sql | 3 +- views/emails/confirm-html.hbs | 110 ++++++++++++++++++++++++++++++--- 8 files changed, 251 insertions(+), 67 deletions(-) diff --git a/lib/mailer.js b/lib/mailer.js index 9ac3f861..6c76fb8f 100644 --- a/lib/mailer.js +++ b/lib/mailer.js @@ -5,6 +5,7 @@ let log = require('npmlog'); let nodemailer = require('nodemailer'); let openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt; let settings = require('./models/settings'); +let tools = require('./tools'); let Handlebars = require('handlebars'); let fs = require('fs'); let path = require('path'); @@ -48,20 +49,30 @@ module.exports.sendMail = (mail, template, callback) => { mail.html = htmlRenderer(template.data || {}); } - getTemplate(template.text, (err, textRenderer) => { + tools.prepareHtml(mail.html, (err, prepareHtml) => { if (err) { - return callback(err); + // ignore } - if (textRenderer) { - mail.text = textRenderer(template.data || {}); - } else if (mail.html) { - mail.text = htmlToText.fromString(mail.html, { - wordwrap: 130 - }); + if (prepareHtml) { + mail.html = prepareHtml; } - module.exports.transport.sendMail(mail, callback); + getTemplate(template.text, (err, textRenderer) => { + if (err) { + return callback(err); + } + + if (textRenderer) { + mail.text = textRenderer(template.data || {}); + } else if (mail.html) { + mail.text = htmlToText.fromString(mail.html, { + wordwrap: 130 + }); + } + + module.exports.transport.sendMail(mail, callback); + }); }); }); diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index e67244b4..efa911a8 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -275,31 +275,43 @@ module.exports.create = (campaign, opts, callback) => { keys.push('cid'); values.push(cid); - db.getConnection((err, connection) => { + tools.prepareHtml(campaign.html, (err, preparedHtml) => { if (err) { - return next(err); + log.error('jsdom', err); + preparedHtml = campaign.html; } - let query = 'INSERT INTO campaigns (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')'; - connection.query(query, values, (err, result) => { - connection.release(); + if (preparedHtml) { + keys.push('html_prepared'); + values.push(preparedHtml); + } + + db.getConnection((err, connection) => { if (err) { return next(err); } - let campaignId = result && result.insertId || false; - if (!campaignId) { - return next(null, false); - } - - // we are going to aqcuire a lot of log info, so we are putting - // sending logs into separate tables - createCampaignTables(campaignId, err => { + let query = 'INSERT INTO campaigns (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')'; + connection.query(query, values, (err, result) => { + connection.release(); if (err) { - // FIXME: rollback return next(err); } - return next(null, campaignId); + + let campaignId = result && result.insertId || false; + if (!campaignId) { + return next(null, false); + } + + // we are going to aqcuire a lot of log info, so we are putting + // sending logs into separate tables + createCampaignTables(campaignId, err => { + if (err) { + // FIXME: rollback + return next(err); + } + return next(null, campaignId); + }); }); }); }); @@ -355,8 +367,8 @@ module.exports.create = (campaign, opts, callback) => { return callback(new Error('Selected template not found')); } - keys = keys.concat(['html', 'text']); - values = values.concat([template.html, template.text]); + campaign.html = template.html; + campaign.text = template.text; create(callback); }); @@ -410,58 +422,70 @@ module.exports.update = (id, updates, callback) => { } }); - db.getConnection((err, connection) => { + tools.prepareHtml(campaign.html, (err, preparedHtml) => { if (err) { - return callback(err); + log.error('jsdom', err); + preparedHtml = campaign.html; } - values.push(id); + if (preparedHtml) { + keys.push('html_prepared'); + values.push(preparedHtml); + } - connection.query('UPDATE campaigns SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1', values, (err, result) => { + db.getConnection((err, connection) => { if (err) { - connection.release(); return callback(err); } - let affected = result && result.affectedRows || false; - if (!affected) { - connection.release(); - return callback(null, affected); - } + values.push(id); - connection.query('SELECT `type`, `source_url` FROM campaigns WHERE id=? LIMIT 1', [id], (err, rows) => { + connection.query('UPDATE campaigns SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1', values, (err, result) => { if (err) { connection.release(); return callback(err); } + let affected = result && result.affectedRows || false; - if (!rows || !rows[0] || rows[0].type !== 2) { - // if not RSS, then nothing to do here + if (!affected) { connection.release(); return callback(null, affected); } - // update seen rss entries to avoid sending old entries to subscribers - feed.fetch(rows[0].source_url, (err, entries) => { + connection.query('SELECT `type`, `source_url` FROM campaigns WHERE id=? LIMIT 1', [id], (err, rows) => { if (err) { connection.release(); return callback(err); } - let query = 'INSERT IGNORE INTO `rss` (`parent`,`guid`,`pubdate`) VALUES ' + entries.map(() => '(?,?,?)').join(','); - - values = []; - entries.forEach(entry => { - values.push(id, entry.guid, entry.date); - }); - - connection.query(query, values, err => { + if (!rows || !rows[0] || rows[0].type !== 2) { + // if not RSS, then nothing to do here connection.release(); - if (err) { - // too late to report as failed - log.error('RSS', err); - } return callback(null, affected); + } + + // update seen rss entries to avoid sending old entries to subscribers + feed.fetch(rows[0].source_url, (err, entries) => { + if (err) { + connection.release(); + return callback(err); + } + + let query = 'INSERT IGNORE INTO `rss` (`parent`,`guid`,`pubdate`) VALUES ' + entries.map(() => '(?,?,?)').join(','); + + values = []; + entries.forEach(entry => { + values.push(id, entry.guid, entry.date); + }); + + connection.query(query, values, err => { + connection.release(); + if (err) { + // too late to report as failed + log.error('RSS', err); + } + return callback(null, affected); + }); }); }); }); diff --git a/lib/tools.js b/lib/tools.js index 6a8a98c8..c194c440 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -4,6 +4,8 @@ let db = require('./db'); let slugify = require('slugify'); let Isemail = require('isemail'); let urllib = require('url'); +let juice = require('juice'); +let jsdom = require('jsdom'); let blockedUsers = ['abuse', 'admin', 'billing', 'compliance', 'devnull', 'dns', 'ftp', 'hostmaster', 'inoc', 'ispfeedback', 'ispsupport', 'listrequest', 'list', 'maildaemon', 'noc', 'noreply', 'noreply', 'null', 'phish', 'phishing', 'postmaster', 'privacy', 'registrar', 'root', 'security', 'spam', 'support', 'sysadmin', 'tech', 'undisclosedrecipients', 'unsubscribe', 'usenet', 'uucp', 'webmaster', 'www']; @@ -16,7 +18,8 @@ module.exports = { updateMenu, validateEmail, formatMessage, - getMessageLinks + getMessageLinks, + prepareHtml }; function toDbKey(key) { @@ -178,3 +181,36 @@ function formatMessage(serviceUrl, campaign, list, subscription, message, filter return value ? filter(value) : match; }); } + +function prepareHtml(html, callback) { + if (!(html || '').toString().trim()) { + return callback(null, false); + } + + jsdom.env(html, (err, win) => { + if (err) { + return callback(err); + } + + let head = win.document.querySelector('head'); + let hasCharsetTag = false; + let metaTags = win.document.querySelectorAll('meta'); + if (metaTags) { + for (let i = 0; i < metaTags.length; i++) { + if (metaTags[i].hasAttribute('charset')) { + metaTags[i].setAttribute('charset', 'utf-8'); + hasCharsetTag = true; + break; + } + } + } + if (!hasCharsetTag) { + let charsetTag = win.document.createElement('meta'); + charsetTag.setAttribute('charset', 'utf-8'); + head.appendChild(charsetTag); + } + let preparedHtml = '' + win.document.documentElement.innerHTML + ''; + + return callback(null, juice(preparedHtml)); + }); +} diff --git a/package.json b/package.json index 8620b56c..cb3232be 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,8 @@ "humanize": "0.0.9", "is-url": "^1.2.1", "isemail": "^2.1.0", + "jsdom": "^8.5.0", + "juice": "^1.10.0", "moment-timezone": "^0.5.3", "morgan": "^1.7.0", "multer": "^1.1.0", diff --git a/services/sender.js b/services/sender.js index d9be0a84..49c773c5 100644 --- a/services/sender.js +++ b/services/sender.js @@ -251,7 +251,7 @@ function formatMessage(message, callback) { renderAndSend(body && body.toString(), '', false); }); } else { - renderAndSend(campaign.html, campaign.text, true); + renderAndSend(campaign.htmlPrepared || campaign.html, campaign.text, true); } }); }); diff --git a/setup/sql/mailtrain.sql b/setup/sql/mailtrain.sql index f9354f77..814ea075 100644 --- a/setup/sql/mailtrain.sql +++ b/setup/sql/mailtrain.sql @@ -29,12 +29,14 @@ CREATE TABLE `campaigns` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `cid` varchar(255) CHARACTER SET ascii NOT NULL, `type` tinyint(4) unsigned NOT NULL DEFAULT '1', + `parent` int(11) unsigned DEFAULT NULL, `name` varchar(255) NOT NULL DEFAULT '', `description` text, `list` int(11) unsigned NOT NULL, `segment` int(11) unsigned DEFAULT NULL, `template` int(11) unsigned NOT NULL, `source_url` varchar(255) CHARACTER SET ascii DEFAULT NULL, + `last_check` timestamp NULL DEFAULT NULL, `from` varchar(255) DEFAULT '', `address` varchar(255) DEFAULT '', `subject` varchar(255) DEFAULT '', @@ -55,7 +57,9 @@ CREATE TABLE `campaigns` ( KEY `name` (`name`(191)), KEY `status` (`status`), KEY `schedule_index` (`scheduled`), - KEY `type_index` (`type`) + KEY `type_index` (`type`), + KEY `parent_index` (`parent`), + KEY `check_index` (`last_check`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `confirmations` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, @@ -137,6 +141,18 @@ CREATE TABLE `lists` ( UNIQUE KEY `cid` (`cid`), KEY `name` (`name`(191)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE `rss` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `parent` int(11) unsigned NOT NULL, + `guid` varchar(255) NOT NULL DEFAULT '', + `pubdate` timestamp NULL DEFAULT NULL, + `campaign` int(11) unsigned DEFAULT NULL, + `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `parent_2` (`parent`,`guid`), + KEY `parent` (`parent`), + CONSTRAINT `rss_ibfk_1` FOREIGN KEY (`parent`) REFERENCES `campaigns` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `segment_rules` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `segment` int(11) unsigned NOT NULL, @@ -163,7 +179,7 @@ CREATE TABLE `settings` ( `value` text NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `key` (`key`) -) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=25 DEFAULT CHARSET=utf8mb4; INSERT INTO `settings` (`id`, `key`, `value`) VALUES (1,'smtp_hostname','localhost'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (2,'smtp_port','465'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (3,'smtp_encryption','TLS'); @@ -180,7 +196,7 @@ INSERT INTO `settings` (`id`, `key`, `value`) VALUES (13,'default_from','My Awes INSERT INTO `settings` (`id`, `key`, `value`) VALUES (14,'default_address','admin@example.com'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (15,'default_subject','Test message'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (16,'default_homepage','http://localhost:3000/'); -INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','7'); +INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','8'); CREATE TABLE `subscription` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `cid` varchar(255) CHARACTER SET ascii NOT NULL, diff --git a/setup/sql/upgrade-00008.sql b/setup/sql/upgrade-00008.sql index 89f3293f..78c30bcf 100644 --- a/setup/sql/upgrade-00008.sql +++ b/setup/sql/upgrade-00008.sql @@ -7,7 +7,7 @@ CREATE TABLE `rss` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `parent` int(11) unsigned NOT NULL, `guid` varchar(255) NOT NULL DEFAULT '', - `pubdate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `pubdate` timestamp NULL DEFAULT NULL, `campaign` int(11) unsigned DEFAULT NULL, `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), @@ -19,6 +19,7 @@ ALTER TABLE `campaigns` ADD COLUMN `parent` int(11) unsigned DEFAULT NULL AFTER CREATE INDEX parent_index ON `campaigns` (`parent`); ALTER TABLE `campaigns` ADD COLUMN `last_check` timestamp NULL DEFAULT NULL AFTER `source_url`; CREATE INDEX check_index ON `campaigns` (`last_check`); +ALTER TABLE `campaigns` ADD COLUMN `html_prepared` text AFTER `html`; # Footer section LOCK TABLES `settings` WRITE; diff --git a/views/emails/confirm-html.hbs b/views/emails/confirm-html.hbs index 54b5a573..29df5045 100644 --- a/views/emails/confirm-html.hbs +++ b/views/emails/confirm-html.hbs @@ -4,23 +4,117 @@ {{title}}: Please Confirm Subscription + + + + -

{{title}}

+
-

Please Confirm Subscription

+

{{title}}

+

Please Confirm Subscription

-
+

Yes, subscribe me to this list

-

+

If you received this email by mistake, simply delete it. You won't be subscribed if you don't click the confirmation link above.

-

If you received this email by mistake, simply delete it. You won't be subscribed if you don't click the confirmation link above.

- -

For questions about this list, please contact: -
{{contactAddress}}

+

For questions about this list, please contact: +
{{contactAddress}}

+
+