From 773977dd96c9e00c0d4cd8dd3203714adf46aa00 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Wed, 25 May 2016 23:58:17 +0300 Subject: [PATCH] v1.10.0 --- CHANGELOG.txt | 6 + lib/caches.js | 28 +++++ lib/models/campaigns.js | 6 +- meta.json | 2 +- package.json | 2 +- services/sender.js | 10 +- setup/sql/mailtrain.sql | 20 ++-- setup/sql/upgrade-00013.sql | 15 +++ views/campaigns/view.hbs | 16 ++- views/lists/view.hbs | 221 ++++++++++++++++++------------------ 10 files changed, 194 insertions(+), 132 deletions(-) create mode 100644 lib/caches.js create mode 100644 setup/sql/upgrade-00013.sql diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a29357aa..49a5a1d6 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,11 @@ # Changelog +## 1.10.0 2016-05-25 + + * Fetch multiple unsent messages at once to speed up delivery + * Fixed a bug of counting unsubscribers correctly + * Use LONGTEXT for template text fields (messages might include inlined images which are larger than 64kB) + ## 1.9.0 2016-05-16 * New look diff --git a/lib/caches.js b/lib/caches.js new file mode 100644 index 00000000..b7c308ba --- /dev/null +++ b/lib/caches.js @@ -0,0 +1,28 @@ +'use strict'; + +let cache = module.exports.cache = new Map(); + +module.exports.push = (name, value) => { + if (!cache.has(name)) { + cache.set(name, []); + } else if (!Array.isArray(cache.get(name))) { + cache.set(name, [].concat(cache.get(name) || [])); + } + cache.get(name).push(value); +}; + +module.exports.shift = name => { + if (!cache.has(name)) { + return false; + } + if (!Array.isArray(cache.get(name))) { + let value = cache.get(name); + cache.delete(name); + return value; + } + let value = cache.get(name).shift(); + if (!cache.get(name).length) { + cache.delete(name); + } + return value; +}; diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index 34816e73..4e0fae42 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -11,6 +11,7 @@ let isUrl = require('is-url'); let feed = require('../feed'); let log = require('npmlog'); let mailer = require('../mailer'); +let caches = require('../caches'); let allowedKeys = ['description', 'from', 'address', 'subject', 'template', 'source_url', 'list', 'segment', 'html', 'text']; @@ -809,6 +810,7 @@ module.exports.pause = (id, callback) => { connection.release(); return callback(err); } + caches.cache.delete('sender queue'); return callback(null, true); }); }); @@ -821,7 +823,7 @@ module.exports.reset = (id, callback) => { return callback(err); } - if (campaign.status !== 3 && !(campaign.status === 2 && campaign.scheduled > new Date())) { + if (campaign.status !== 3 && campaign.status !== 4 && !(campaign.status === 2 && campaign.scheduled > new Date())) { return callback(null, false); } @@ -835,6 +837,8 @@ module.exports.reset = (id, callback) => { connection.release(); return callback(err); } + + caches.cache.delete('sender queue'); connection.query('DELETE FROM links WHERE campaign=?', [id], err => { if (err) { connection.release(); diff --git a/meta.json b/meta.json index 43faa406..bca525e4 100644 --- a/meta.json +++ b/meta.json @@ -1,3 +1,3 @@ { - "schemaVersion": 12 + "schemaVersion": 13 } diff --git a/package.json b/package.json index 9ee31c0c..5eb2d747 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mailtrain", "private": true, - "version": "1.9.0", + "version": "1.10.0", "description": "Self hosted email newsletter app", "main": "index.js", "scripts": { diff --git a/services/sender.js b/services/sender.js index 1e10e7e3..19506e10 100644 --- a/services/sender.js +++ b/services/sender.js @@ -15,9 +15,7 @@ let shortid = require('shortid'); let url = require('url'); let htmlToText = require('html-to-text'); let request = require('request'); - -// to speed things up fetch several unsent messages and store these into a cache -let fetchCache = []; +let caches = require('../lib/caches'); function findUnsent(callback) { @@ -49,8 +47,8 @@ function findUnsent(callback) { }); }; - if (fetchCache.length) { - let cached = fetchCache.shift(); + if (caches.cache.has('sender queue')) { + let cached = caches.shift('sender queue'); return returnUnsent(cached.row, cached.campaign); } @@ -125,7 +123,7 @@ function findUnsent(callback) { connection.release(); rows.forEach(row => { - fetchCache.push({ + caches.push('sender queue', { row, campaign }); diff --git a/setup/sql/mailtrain.sql b/setup/sql/mailtrain.sql index fe1adaa0..ecd895f8 100644 --- a/setup/sql/mailtrain.sql +++ b/setup/sql/mailtrain.sql @@ -14,7 +14,9 @@ CREATE TABLE `campaign` ( PRIMARY KEY (`id`), UNIQUE KEY `list` (`list`,`segment`,`subscription`), KEY `created` (`created`), - KEY `response_id` (`response_id`) + KEY `response_id` (`response_id`), + KEY `status_index` (`status`), + KEY `subscription_index` (`subscription`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `campaign_tracker` ( `list` int(11) unsigned NOT NULL, @@ -43,9 +45,9 @@ CREATE TABLE `campaigns` ( `from` varchar(255) DEFAULT '', `address` varchar(255) DEFAULT '', `subject` varchar(255) DEFAULT '', - `html` text, - `html_prepared` text, - `text` text, + `html` longtext, + `html_prepared` longtext, + `text` longtext, `status` tinyint(4) unsigned NOT NULL DEFAULT '1', `scheduled` timestamp NULL DEFAULT NULL, `status_change` timestamp NULL DEFAULT NULL, @@ -174,7 +176,7 @@ CREATE TABLE `segments` ( `created` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `list` (`list`), - KEY `name` (`name`(191)), + KEY `name` (`name`), CONSTRAINT `segments_ibfk_1` FOREIGN KEY (`list`) REFERENCES `lists` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CREATE TABLE `settings` ( @@ -183,7 +185,7 @@ CREATE TABLE `settings` ( `value` text NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `key` (`key`) -) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=30 DEFAULT CHARSET=utf8mb4; INSERT INTO `settings` (`id`, `key`, `value`) VALUES (1,'smtp_hostname','localhost'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (2,'smtp_port','465'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (3,'smtp_encryption','TLS'); @@ -200,7 +202,7 @@ INSERT INTO `settings` (`id`, `key`, `value`) VALUES (13,'default_from','My Awes INSERT INTO `settings` (`id`, `key`, `value`) VALUES (14,'default_address','admin@example.com'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (15,'default_subject','Test message'); INSERT INTO `settings` (`id`, `key`, `value`) VALUES (16,'default_homepage','http://localhost:3000/'); -INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','10'); +INSERT INTO `settings` (`id`, `key`, `value`) VALUES (17,'db_schema_version','13'); CREATE TABLE `subscription` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `cid` varchar(255) CHARACTER SET ascii NOT NULL, @@ -228,8 +230,8 @@ CREATE TABLE `templates` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL DEFAULT '', `description` text, - `html` text, - `text` text, + `html` longtext, + `text` longtext, `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `name` (`name`(191)) diff --git a/setup/sql/upgrade-00013.sql b/setup/sql/upgrade-00013.sql new file mode 100644 index 00000000..288e8e21 --- /dev/null +++ b/setup/sql/upgrade-00013.sql @@ -0,0 +1,15 @@ +# Header section +# Define incrementing schema version number +SET @schema_version = '13'; + +-- {{#each tables.campaign}} + + # Adds separate index for 'subscription' on campaign messages table + CREATE INDEX subscription_index ON `{{this}}` (`subscription`); + +-- {{/each}} + +# Footer section +LOCK TABLES `settings` WRITE; +INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version; +UNLOCK TABLES; diff --git a/views/campaigns/view.hbs b/views/campaigns/view.hbs index cbe84648..c736592c 100644 --- a/views/campaigns/view.hbs +++ b/views/campaigns/view.hbs @@ -210,13 +210,21 @@ {{#if isPaused}}
-
+ - -
+ +
+ + +
+ + + +

Sending paused

{{/if}} diff --git a/views/lists/view.hbs b/views/lists/view.hbs index 9dd0a5ee..c535b167 100644 --- a/views/lists/view.hbs +++ b/views/lists/view.hbs @@ -61,140 +61,141 @@
+ {{#if showSubscriptions}} +

-

+
+ + + + + + + + {{#each customFields}} + + {{/each}} + + + + +
+ # + + Address + + First Name + + Last Name + + {{name}} + + Status +
+
-
- - - + {{/if}} + +
+ {{#if showImports}} +

+ +
+
+ + + + - {{#each customFields}} - - {{/each}} - - - -
# - Address + Created - First Name + Finished - Last Name + Type + + Added + + Updated + + Failed - {{name}} - Status
-
+ -
-
+ + + + {{#if imports}} + {{#each imports}} + + + {{index}} + -

+ + {{created}} + -
- - - - - - - - - - - - - - - {{#if imports}} - {{#each imports}} + + + + + + + + {{/each}} + {{else}} - - - - - - - - - - - - - {{/each}} - {{else}} - - - - {{/if}} - -
- # - - Created - - Finished - - Type - - Added - - Updated - - Failed - - Status - + + {{#if finished}} + {{finished}} + {{else}} + No + {{/if}} +
+ {{importType}} + + {{new}} + + {{updated}} + + {{#if failed}}{{failed}}{{else}}0 {{/if}} + + {{#if error}} + {{importStatus}} + {{else}} + {{importStatus}} + {{/if}} + +
+ + + + +
+
- {{index}} - - {{created}} - - {{#if finished}} - {{finished}} - {{else}} - No - {{/if}} - - {{importType}} - - {{new}} - - {{updated}} - - {{#if failed}}{{failed}}{{else}}0 {{/if}} - - {{#if error}} - {{importStatus}} - {{else}} - {{importStatus}} - {{/if}} - -
- - - - -
+ 		+ No data available in table
- No data available in table -
-
- + {{/if}} + + +
+ {{/if}}