Use juice to prepare html messages

lib/mailer.js

@@ -5,6 +5,7 @@ let log = require('npmlog');
 let nodemailer = require('nodemailer');
 let openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
 let settings = require('./models/settings');
+let tools = require('./tools');
 let Handlebars = require('handlebars');
 let fs = require('fs');
 let path = require('path');
@@ -48,6 +49,15 @@ module.exports.sendMail = (mail, template, callback) => {
             mail.html = htmlRenderer(template.data || {});
         }
 
+        tools.prepareHtml(mail.html, (err, prepareHtml) => {
+            if (err) {
+                // ignore
+            }
+
+            if (prepareHtml) {
+                mail.html = prepareHtml;
+            }
+
             getTemplate(template.text, (err, textRenderer) => {
                 if (err) {
                     return callback(err);
@@ -64,6 +74,7 @@ module.exports.sendMail = (mail, template, callback) => {
                 module.exports.transport.sendMail(mail, callback);
             });
         });
+    });
 
 };
 

lib/models/campaigns.js

@@ -275,6 +275,17 @@ module.exports.create = (campaign, opts, callback) => {
             keys.push('cid');
             values.push(cid);
 
+            tools.prepareHtml(campaign.html, (err, preparedHtml) => {
+                if (err) {
+                    log.error('jsdom', err);
+                    preparedHtml = campaign.html;
+                }
+
+                if (preparedHtml) {
+                    keys.push('html_prepared');
+                    values.push(preparedHtml);
+                }
+
                 db.getConnection((err, connection) => {
                     if (err) {
                         return next(err);
@@ -303,6 +314,7 @@ module.exports.create = (campaign, opts, callback) => {
                         });
                     });
                 });
+            });
         };
 
         if (campaign.type === 2) {
@@ -355,8 +367,8 @@ module.exports.create = (campaign, opts, callback) => {
                     return callback(new Error('Selected template not found'));
                 }
 
-                keys = keys.concat(['html', 'text']);
-                values = values.concat([template.html, template.text]);
+                campaign.html = template.html;
+                campaign.text = template.text;
 
                 create(callback);
             });
@@ -410,6 +422,17 @@ module.exports.update = (id, updates, callback) => {
             }
         });
 
+        tools.prepareHtml(campaign.html, (err, preparedHtml) => {
+            if (err) {
+                log.error('jsdom', err);
+                preparedHtml = campaign.html;
+            }
+
+            if (preparedHtml) {
+                keys.push('html_prepared');
+                values.push(preparedHtml);
+            }
+
             db.getConnection((err, connection) => {
                 if (err) {
                     return callback(err);
@@ -468,6 +491,7 @@ module.exports.update = (id, updates, callback) => {
                 });
             });
         });
+    });
 };
 
 module.exports.delete = (id, callback) => {

lib/tools.js

@@ -4,6 +4,8 @@ let db = require('./db');
 let slugify = require('slugify');
 let Isemail = require('isemail');
 let urllib = require('url');
+let juice = require('juice');
+let jsdom = require('jsdom');
 
 let blockedUsers = ['abuse', 'admin', 'billing', 'compliance', 'devnull', 'dns', 'ftp', 'hostmaster', 'inoc', 'ispfeedback', 'ispsupport', 'listrequest', 'list', 'maildaemon', 'noc', 'noreply', 'noreply', 'null', 'phish', 'phishing', 'postmaster', 'privacy', 'registrar', 'root', 'security', 'spam', 'support', 'sysadmin', 'tech', 'undisclosedrecipients', 'unsubscribe', 'usenet', 'uucp', 'webmaster', 'www'];
 
@@ -16,7 +18,8 @@ module.exports = {
     updateMenu,
     validateEmail,
     formatMessage,
-    getMessageLinks
+    getMessageLinks,
+    prepareHtml
 };
 
 function toDbKey(key) {
@@ -178,3 +181,36 @@ function formatMessage(serviceUrl, campaign, list, subscription, message, filter
         return value ? filter(value) : match;
     });
 }
+
+function prepareHtml(html, callback) {
+    if (!(html || '').toString().trim()) {
+        return callback(null, false);
+    }
+
+    jsdom.env(html, (err, win) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let head = win.document.querySelector('head');
+        let hasCharsetTag = false;
+        let metaTags = win.document.querySelectorAll('meta');
+        if (metaTags) {
+            for (let i = 0; i < metaTags.length; i++) {
+                if (metaTags[i].hasAttribute('charset')) {
+                    metaTags[i].setAttribute('charset', 'utf-8');
+                    hasCharsetTag = true;
+                    break;
+                }
+            }
+        }
+        if (!hasCharsetTag) {
+            let charsetTag = win.document.createElement('meta');
+            charsetTag.setAttribute('charset', 'utf-8');
+            head.appendChild(charsetTag);
+        }
+        let preparedHtml = '<!doctype html><html>' + win.document.documentElement.innerHTML + '</html>';
+
+        return callback(null, juice(preparedHtml));
+    });
+}

package.json

@@ -50,6 +50,8 @@
     "humanize": "0.0.9",
     "is-url": "^1.2.1",
     "isemail": "^2.1.0",
+    "jsdom": "^8.5.0",
+    "juice": "^1.10.0",
     "moment-timezone": "^0.5.3",
     "morgan": "^1.7.0",
     "multer": "^1.1.0",

services/sender.js

@@ -251,7 +251,7 @@ function formatMessage(message, callback) {
                             renderAndSend(body && body.toString(), '', false);
                         });
                     } else {
-                        renderAndSend(campaign.html, campaign.text, true);
+                        renderAndSend(campaign.htmlPrepared || campaign.html, campaign.text, true);
                     }
                 });
             });

setup/sql/mailtrain.sql

@@ -29,12 +29,14 @@ CREATE TABLE `campaigns` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `cid` varchar(255) CHARACTER SET ascii NOT NULL,
   `type` tinyint(4) unsigned NOT NULL DEFAULT '1',
+  `parent` int(11) unsigned DEFAULT NULL,
   `name` varchar(255) NOT NULL DEFAULT '',
   `description` text,
   `list` int(11) unsigned NOT NULL,
   `segment` int(11) unsigned DEFAULT NULL,
   `template` int(11) unsigned NOT NULL,
   `source_url` varchar(255) CHARACTER SET ascii DEFAULT NULL,
+  `last_check` timestamp NULL DEFAULT NULL,
   `from` varchar(255) DEFAULT '',
   `address` varchar(255) DEFAULT '',
   `subject` varchar(255) DEFAULT '',
@@ -55,7 +57,9 @@ CREATE TABLE `campaigns` (
   KEY `name` (`name`(191)),
   KEY `status` (`status`),
   KEY `schedule_index` (`scheduled`),
-  KEY `type_index` (`type`)
+  KEY `type_index` (`type`),
+  KEY `parent_index` (`parent`),
+  KEY `check_index` (`last_check`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 CREATE TABLE `confirmations` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
@@ -137,6 +141,18 @@ CREATE TABLE `lists` (
   UNIQUE KEY `cid` (`cid`),
   KEY `name` (`name`(191))
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+CREATE TABLE `rss` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `parent` int(11) unsigned NOT NULL,
+  `guid` varchar(255) NOT NULL DEFAULT '',
+  `pubdate` timestamp NULL DEFAULT NULL,
+  `campaign` int(11) unsigned DEFAULT NULL,
+  `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `parent_2` (`parent`,`guid`),
+  KEY `parent` (`parent`),
+  CONSTRAINT `rss_ibfk_1` FOREIGN KEY (`parent`) REFERENCES `campaigns` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 CREATE TABLE `segment_rules` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `segment` int(11) unsigned NOT NULL,
@@ -163,7 +179,7 @@ CREATE TABLE `settings` (
   `value` text NOT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `key` (`key`)
-) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8mb4;
+) ENGINE=InnoDB AUTO_INCREMENT=25 DEFAULT CHARSET=utf8mb4;
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (1,'smtp_hostname','localhost');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (2,'smtp_port','465');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (3,'smtp_encryption','TLS');
@@ -180,7 +196,7 @@ INSERT INTO `settings` (`id`, `key`, `value`) VALUES (13,'default_from','My Awes
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (14,'default_address','admin@example.com');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (15,'default_subject','Test message');
 INSERT INTO `settings` (`id`, `key`, `value`) VALUES (16,'default_homepage','http://localhost:3000/');
-INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','7');
+INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','8');
 CREATE TABLE `subscription` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `cid` varchar(255) CHARACTER SET ascii NOT NULL,

setup/sql/upgrade-00008.sql

@@ -7,7 +7,7 @@ CREATE TABLE `rss` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `parent` int(11) unsigned NOT NULL,
   `guid` varchar(255) NOT NULL DEFAULT '',
-  `pubdate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `pubdate` timestamp NULL DEFAULT NULL,
   `campaign` int(11) unsigned DEFAULT NULL,
   `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
@@ -19,6 +19,7 @@ ALTER TABLE `campaigns` ADD COLUMN `parent` int(11) unsigned DEFAULT NULL AFTER
 CREATE INDEX parent_index ON `campaigns` (`parent`);
 ALTER TABLE `campaigns` ADD COLUMN `last_check` timestamp NULL DEFAULT NULL AFTER `source_url`;
 CREATE INDEX check_index ON `campaigns` (`last_check`);
+ALTER TABLE `campaigns` ADD COLUMN `html_prepared` text AFTER `html`;
 
 # Footer section
 LOCK TABLES `settings` WRITE;