Use juice to prepare html messages

lib/mailer.js

@@ -5,6 +5,7 @@ let log = require('npmlog');
 let nodemailer = require('nodemailer');
 let openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
 let settings = require('./models/settings');
+let tools = require('./tools');
 let Handlebars = require('handlebars');
 let fs = require('fs');
 let path = require('path');
@@ -48,20 +49,30 @@ module.exports.sendMail = (mail, template, callback) => {
             mail.html = htmlRenderer(template.data || {});
         }
 
-        getTemplate(template.text, (err, textRenderer) => {
+        tools.prepareHtml(mail.html, (err, prepareHtml) => {
             if (err) {
-                return callback(err);
+                // ignore
             }
 
-            if (textRenderer) {
+            if (prepareHtml) {
-                mail.text = textRenderer(template.data || {});
+                mail.html = prepareHtml;
-            } else if (mail.html) {
-                mail.text = htmlToText.fromString(mail.html, {
-                    wordwrap: 130
-                });
             }
 
-            module.exports.transport.sendMail(mail, callback);
+            getTemplate(template.text, (err, textRenderer) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                if (textRenderer) {
+                    mail.text = textRenderer(template.data || {});
+                } else if (mail.html) {
+                    mail.text = htmlToText.fromString(mail.html, {
+                        wordwrap: 130
+                    });
+                }
+
+                module.exports.transport.sendMail(mail, callback);
+            });
         });
     });
 

lib/models/campaigns.js

@@ -275,31 +275,43 @@ module.exports.create = (campaign, opts, callback) => {
             keys.push('cid');
             values.push(cid);
 
-            db.getConnection((err, connection) => {
+            tools.prepareHtml(campaign.html, (err, preparedHtml) => {
                 if (err) {
-                    return next(err);
+                    log.error('jsdom', err);
+                    preparedHtml = campaign.html;
                 }
 
-                let query = 'INSERT INTO campaigns (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
+                if (preparedHtml) {
-                connection.query(query, values, (err, result) => {
+                    keys.push('html_prepared');
-                    connection.release();
+                    values.push(preparedHtml);
+                }
+
+                db.getConnection((err, connection) => {
                     if (err) {
                         return next(err);
                     }
 
-                    let campaignId = result && result.insertId || false;
+                    let query = 'INSERT INTO campaigns (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
-                    if (!campaignId) {
+                    connection.query(query, values, (err, result) => {
-                        return next(null, false);
+                        connection.release();
-                    }
-
-                    // we are going to aqcuire a lot of log info, so we are putting
-                    // sending logs into separate tables
-                    createCampaignTables(campaignId, err => {
                         if (err) {
-                            // FIXME: rollback
                             return next(err);
                         }
-                        return next(null, campaignId);
+
+                        let campaignId = result && result.insertId || false;
+                        if (!campaignId) {
+                            return next(null, false);
+                        }
+
+                        // we are going to aqcuire a lot of log info, so we are putting
+                        // sending logs into separate tables
+                        createCampaignTables(campaignId, err => {
+                            if (err) {
+                                // FIXME: rollback
+                                return next(err);
+                            }
+                            return next(null, campaignId);
+                        });
                     });
                 });
             });
@@ -355,8 +367,8 @@ module.exports.create = (campaign, opts, callback) => {
                     return callback(new Error('Selected template not found'));
                 }
 
-                keys = keys.concat(['html', 'text']);
+                campaign.html = template.html;
-                values = values.concat([template.html, template.text]);
+                campaign.text = template.text;
 
                 create(callback);
             });
@@ -410,58 +422,70 @@ module.exports.update = (id, updates, callback) => {
             }
         });
 
-        db.getConnection((err, connection) => {
+        tools.prepareHtml(campaign.html, (err, preparedHtml) => {
             if (err) {
-                return callback(err);
+                log.error('jsdom', err);
+                preparedHtml = campaign.html;
             }
 
-            values.push(id);
+            if (preparedHtml) {
+                keys.push('html_prepared');
+                values.push(preparedHtml);
+            }
 
-            connection.query('UPDATE campaigns SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1', values, (err, result) => {
+            db.getConnection((err, connection) => {
                 if (err) {
-                    connection.release();
                     return callback(err);
                 }
-                let affected = result && result.affectedRows || false;
 
-                if (!affected) {
+                values.push(id);
-                    connection.release();
-                    return callback(null, affected);
-                }
 
-                connection.query('SELECT `type`, `source_url` FROM campaigns WHERE id=? LIMIT 1', [id], (err, rows) => {
+                connection.query('UPDATE campaigns SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1', values, (err, result) => {
                     if (err) {
                         connection.release();
                         return callback(err);
                     }
+                    let affected = result && result.affectedRows || false;
 
-                    if (!rows || !rows[0] || rows[0].type !== 2) {
+                    if (!affected) {
-                        // if not RSS, then nothing to do here
                         connection.release();
                         return callback(null, affected);
                     }
 
-                    // update seen rss entries to avoid sending old entries to subscribers
+                    connection.query('SELECT `type`, `source_url` FROM campaigns WHERE id=? LIMIT 1', [id], (err, rows) => {
-                    feed.fetch(rows[0].source_url, (err, entries) => {
                         if (err) {
                             connection.release();
                             return callback(err);
                         }
 
-                        let query = 'INSERT IGNORE INTO `rss` (`parent`,`guid`,`pubdate`) VALUES ' + entries.map(() => '(?,?,?)').join(',');
+                        if (!rows || !rows[0] || rows[0].type !== 2) {
-
+                            // if not RSS, then nothing to do here
-                        values = [];
-                        entries.forEach(entry => {
-                            values.push(id, entry.guid, entry.date);
-                        });
-
-                        connection.query(query, values, err => {
                             connection.release();
-                            if (err) {
-                                // too late to report as failed
-                                log.error('RSS', err);
-                            }
                             return callback(null, affected);
+                        }
+
+                        // update seen rss entries to avoid sending old entries to subscribers
+                        feed.fetch(rows[0].source_url, (err, entries) => {
+                            if (err) {
+                                connection.release();
+                                return callback(err);
+                            }
+
+                            let query = 'INSERT IGNORE INTO `rss` (`parent`,`guid`,`pubdate`) VALUES ' + entries.map(() => '(?,?,?)').join(',');
+
+                            values = [];
+                            entries.forEach(entry => {
+                                values.push(id, entry.guid, entry.date);
+                            });
+
+                            connection.query(query, values, err => {
+                                connection.release();
+                                if (err) {
+                                    // too late to report as failed
+                                    log.error('RSS', err);
+                                }
+                                return callback(null, affected);
+                            });
                         });
                     });
                 });

lib/tools.js

@@ -4,6 +4,8 @@ let db = require('./db');
 let slugify = require('slugify');
 let Isemail = require('isemail');
 let urllib = require('url');
+let juice = require('juice');
+let jsdom = require('jsdom');
 
 let blockedUsers = ['abuse', 'admin', 'billing', 'compliance', 'devnull', 'dns', 'ftp', 'hostmaster', 'inoc', 'ispfeedback', 'ispsupport', 'listrequest', 'list', 'maildaemon', 'noc', 'noreply', 'noreply', 'null', 'phish', 'phishing', 'postmaster', 'privacy', 'registrar', 'root', 'security', 'spam', 'support', 'sysadmin', 'tech', 'undisclosedrecipients', 'unsubscribe', 'usenet', 'uucp', 'webmaster', 'www'];
 
@@ -16,7 +18,8 @@ module.exports = {
     updateMenu,
     validateEmail,
     formatMessage,
-    getMessageLinks
+    getMessageLinks,
+    prepareHtml
 };
 
 function toDbKey(key) {
@@ -178,3 +181,36 @@ function formatMessage(serviceUrl, campaign, list, subscription, message, filter
         return value ? filter(value) : match;
     });
 }
+
+function prepareHtml(html, callback) {
+    if (!(html || '').toString().trim()) {
+        return callback(null, false);
+    }
+
+    jsdom.env(html, (err, win) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let head = win.document.querySelector('head');
+        let hasCharsetTag = false;
+        let metaTags = win.document.querySelectorAll('meta');
+        if (metaTags) {
+            for (let i = 0; i < metaTags.length; i++) {
+                if (metaTags[i].hasAttribute('charset')) {
+                    metaTags[i].setAttribute('charset', 'utf-8');
+                    hasCharsetTag = true;
+                    break;
+                }
+            }
+        }
+        if (!hasCharsetTag) {
+            let charsetTag = win.document.createElement('meta');
+            charsetTag.setAttribute('charset', 'utf-8');
+            head.appendChild(charsetTag);
+        }
+        let preparedHtml = '<!doctype html><html>' + win.document.documentElement.innerHTML + '</html>';
+
+        return callback(null, juice(preparedHtml));
+    });
+}

package.json

@@ -50,6 +50,8 @@
     "humanize": "0.0.9",
     "is-url": "^1.2.1",
     "isemail": "^2.1.0",
+    "jsdom": "^8.5.0",
+    "juice": "^1.10.0",
     "moment-timezone": "^0.5.3",
     "morgan": "^1.7.0",
     "multer": "^1.1.0",

services/sender.js

@@ -251,7 +251,7 @@ function formatMessage(message, callback) {
                             renderAndSend(body && body.toString(), '', false);
                         });
                     } else {
-                        renderAndSend(campaign.html, campaign.text, true);
+                        renderAndSend(campaign.htmlPrepared || campaign.html, campaign.text, true);
                     }
                 });
             });

setup/sql/mailtrain.sql

@@ -29,12 +29,14 @@ CREATE TABLE `campaigns` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `cid` varchar(255) CHARACTER SET ascii NOT NULL,
   `type` tinyint(4) unsigned NOT NULL DEFAULT '1',
+  `parent` int(11) unsigned DEFAULT NULL,
   `name` varchar(255) NOT NULL DEFAULT '',
   `description` text,
   `list` int(11) unsigned NOT NULL,
   `segment` int(11) unsigned DEFAULT NULL,
   `template` int(11) unsigned NOT NULL,
   `source_url` varchar(255) CHARACTER SET ascii DEFAULT NULL,
+  `last_check` timestamp NULL DEFAULT NULL,
   `from` varchar(255) DEFAULT '',
   `address` varchar(255) DEFAULT '',
   `subject` varchar(255) DEFAULT '',
@@ -55,7 +57,9 @@ CREATE TABLE `campaigns` (
   KEY `name` (`name`(191)),
   KEY `status` (`status`),
   KEY `schedule_index` (`scheduled`),
-  KEY `type_index` (`type`)
+  KEY `type_index` (`type`),
+  KEY `parent_index` (`parent`),
+  KEY `check_index` (`last_check`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 CREATE TABLE `confirmations` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
@@ -137,6 +141,18 @@ CREATE TABLE `lists` (
   UNIQUE KEY `cid` (`cid`),
   KEY `name` (`name`(191))
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+CREATE TABLE `rss` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `parent` int(11) unsigned NOT NULL,
+  `guid` varchar(255) NOT NULL DEFAULT '',
+  `pubdate` timestamp NULL DEFAULT NULL,
+  `campaign` int(11) unsigned DEFAULT NULL,
+  `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `parent_2` (`parent`,`guid`),
+  KEY `parent` (`parent`),
+  CONSTRAINT `rss_ibfk_1` FOREIGN KEY (`parent`) REFERENCES `campaigns` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 CREATE TABLE `segment_rules` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `segment` int(11) unsigned NOT NULL,
@@ -163,7 +179,7 @@ CREATE TABLE `settings` (
   `value` text NOT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `key` (`key`)
-) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8mb4;
+) ENGINE=InnoDB AUTO_INCREMENT=25 DEFAULT CHARSET=utf8mb4;
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (1,'smtp_hostname','localhost');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (2,'smtp_port','465');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (3,'smtp_encryption','TLS');
@@ -180,7 +196,7 @@ INSERT INTO `settings` (`id`, `key`, `value`) VALUES (13,'default_from','My Awes
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (14,'default_address','admin@example.com');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (15,'default_subject','Test message');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (16,'default_homepage','http://localhost:3000/');
-INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','7');
+INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','8');
 CREATE TABLE `subscription` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `cid` varchar(255) CHARACTER SET ascii NOT NULL,

setup/sql/upgrade-00008.sql

@@ -7,7 +7,7 @@ CREATE TABLE `rss` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `parent` int(11) unsigned NOT NULL,
   `guid` varchar(255) NOT NULL DEFAULT '',
-  `pubdate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `pubdate` timestamp NULL DEFAULT NULL,
   `campaign` int(11) unsigned DEFAULT NULL,
   `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
@@ -19,6 +19,7 @@ ALTER TABLE `campaigns` ADD COLUMN `parent` int(11) unsigned DEFAULT NULL AFTER
 CREATE INDEX parent_index ON `campaigns` (`parent`);
 ALTER TABLE `campaigns` ADD COLUMN `last_check` timestamp NULL DEFAULT NULL AFTER `source_url`;
 CREATE INDEX check_index ON `campaigns` (`last_check`);
+ALTER TABLE `campaigns` ADD COLUMN `html_prepared` text AFTER `html`;
 
 # Footer section
 LOCK TABLES `settings` WRITE;