From fd0e75da27b64212f171f4ae07d6ecabd92aebcd Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 3 May 2016 12:36:06 +0300 Subject: [PATCH] added view for RSS campaigns --- config/default.toml | 2 +- lib/feed.js | 26 ++-- lib/models/campaigns.js | 195 +++++++++++++++++++++--- meta.json | 2 +- package.json | 1 + routes/campaigns.js | 39 ++++- setup/sql/upgrade-00008.sql | 24 +++ views/campaigns/create-rss.hbs | 2 +- views/campaigns/edit-rss.hbs | 2 +- views/campaigns/view.hbs | 271 +++++++++++++++++++++------------ 10 files changed, 430 insertions(+), 134 deletions(-) create mode 100644 setup/sql/upgrade-00008.sql diff --git a/config/default.toml b/config/default.toml index ed9f1143..64af43c9 100644 --- a/config/default.toml +++ b/config/default.toml @@ -32,7 +32,7 @@ user="mailtrain" password="mailtrain" database="mailtrain" charset="utf8mb4" -timezone="UTC" +timezone="local" [redis] # enable to use Redis session cache or disable if Redis is not installed diff --git a/lib/feed.js b/lib/feed.js index 54c345fb..fbbe1a57 100644 --- a/lib/feed.js +++ b/lib/feed.js @@ -7,6 +7,7 @@ module.exports.fetch = (url, callback) => { let req = request(url); let feedparser = new FeedParser(); let returned = false; + let entries = []; req.setHeader('user-agent', 'Mailtrain'); req.setHeader('accept', 'text/html,application/xhtml+xml'); @@ -40,17 +41,24 @@ module.exports.fetch = (url, callback) => { }); feedparser.on('readable', () => { - // This is where the action is! - let meta = feedparser.meta; let item; - while ((item = feedparser.read())) { - //console.log(require('util').inspect(item, false, 22)); - console.log(item.title); - console.log(item.description || item.summary); - console.log('--------'); + let entry = { + title: item.title, + date: item.date || item.pubdate || item.pubDate || new Date(), + guid: item.guid || item.link, + link: item.link, + content: item.description || item.summary + }; + entries.push(entry); } }); -}; -module.exports.fetch('https://andris9.wordpress.com/feed/', console.log); + feedparser.on('end', () => { + if (returned) { + return; + } + returned = true; + callback(null, entries); + }); +}; diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index bacae16e..c2f0ccd4 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -7,6 +7,9 @@ let templates = require('./templates'); let segments = require('./segments'); let subscriptions = require('./subscriptions'); let shortid = require('shortid'); +let isUrl = require('is-url'); +let feed = require('../feed'); +let log = require('npmlog'); let allowedKeys = ['description', 'from', 'address', 'subject', 'template', 'source_url', 'list', 'segment', 'html', 'text']; @@ -32,7 +35,7 @@ module.exports.list = (start, limit, callback) => { }); }; -module.exports.filter = (request, callback) => { +module.exports.filter = (request, parent, callback) => { let columns = ['#', 'name', 'description', 'status', 'created']; let processQuery = queryData => { @@ -107,11 +110,20 @@ module.exports.filter = (request, callback) => { }); }; - processQuery({ - // only find normal campaigns at this point - where: '`type`=?', - values: [1] - }); + if (parent) { + processQuery({ + // only find normal and RSS parent campaigns at this point + where: '`parent`=?', + values: [parent] + }); + } else { + + processQuery({ + // only find normal and RSS parent campaigns at this point + where: '`type` IN (?,?)', + values: [1, 2] + }); + } }; module.exports.getByCid = (cid, callback) => { @@ -215,6 +227,10 @@ module.exports.create = (campaign, callback) => { return callback(new Error('Campaign Name must be set')); } + if (campaign.type === 2 && !campaign.sourceUrl || !isUrl(campaign.sourceUrl)) { + return callback(new Error('RSS URL must be set')); + } + lists.get(campaign.list, (err, list) => { if (err) { return callback(err); @@ -223,10 +239,15 @@ module.exports.create = (campaign, callback) => { return callback(new Error('Selected list not found')); } - let keys = ['name']; - let values = [name]; + let keys = ['name', 'type']; + let values = [name, campaign.type]; - let create = () => { + if (campaign.type === 2) { + keys.push('status'); + values.push(5); // inactive + } + + let create = next => { Object.keys(campaign).forEach(key => { let value = typeof campaign[key] === 'number' ? campaign[key] : (campaign[key] || '').toString().trim(); key = tools.toDbKey(key); @@ -242,19 +263,19 @@ module.exports.create = (campaign, callback) => { db.getConnection((err, connection) => { if (err) { - return callback(err); + return next(err); } let query = 'INSERT INTO campaigns (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')'; connection.query(query, values, (err, result) => { connection.release(); if (err) { - return callback(err); + return next(err); } let campaignId = result && result.insertId || false; if (!campaignId) { - return callback(null, false); + return next(null, false); } // we are going to aqcuire a lot of log info, so we are putting @@ -262,16 +283,55 @@ module.exports.create = (campaign, callback) => { createCampaignTables(campaignId, err => { if (err) { // FIXME: rollback - return callback(err); + return next(err); } - return callback(null, campaignId); + return next(null, campaignId); }); }); }); }; if (campaign.type === 2) { - create(); + feed.fetch(campaign.sourceUrl, (err, entries) => { + if (err) { + return callback(err); + } + + create((err, campaignId) => { + if (err) { + return callback(err); + } + if (!campaignId || !entries.length) { + return callback(null, campaignId); + } + + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + // store references to already existing feed entries + // this is needed to detect new entries + let query = 'INSERT IGNORE INTO `rss` (`parent`,`guid`,`pubdate`) VALUES ' + entries.map(() => '(?,?,?)').join(','); + + values = []; + entries.forEach(entry => { + values.push(campaignId, entry.guid, entry.date); + }); + + connection.query(query, values, err => { + connection.release(); + if (err) { + // too late to report as failed + log.error('RSS', err); + } + + return callback(null, campaignId); + }); + }); + }); + }); + return; } else if (campaign.template) { templates.get(campaign.template, (err, template) => { if (err) { @@ -284,10 +344,11 @@ module.exports.create = (campaign, callback) => { keys = keys.concat(['html', 'text']); values = values.concat([template.html, template.text]); - create(); + create(callback); }); + return; } else { - create(); + return create(callback); } }); }; @@ -343,11 +404,53 @@ module.exports.update = (id, updates, callback) => { values.push(id); connection.query('UPDATE campaigns SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1', values, (err, result) => { - connection.release(); if (err) { + connection.release(); return callback(err); } - return callback(null, result && result.affectedRows || false); + let affected = result && result.affectedRows || false; + + if (!affected) { + connection.release(); + return callback(null, affected); + } + + connection.query('SELECT `type`, `source_url` FROM campaigns WHERE id=? LIMIT 1', [id], (err, rows) => { + if (err) { + connection.release(); + return callback(err); + } + + if (!rows || !rows[0] || rows[0].type !== 2) { + // if not RSS, then nothing to do here + connection.release(); + return callback(null, affected); + } + + // update seen rss entries to avoid sending old entries to subscribers + feed.fetch(rows[0].source_url, (err, entries) => { + if (err) { + connection.release(); + return callback(err); + } + + let query = 'INSERT IGNORE INTO `rss` (`parent`,`guid`,`pubdate`) VALUES ' + entries.map(() => '(?,?,?)').join(','); + + values = []; + entries.forEach(entry => { + values.push(id, entry.guid, entry.date); + }); + + connection.query(query, values, err => { + connection.release(); + if (err) { + // too late to report as failed + log.error('RSS', err); + } + return callback(null, affected); + }); + }); + }); }); }); }); @@ -498,6 +601,60 @@ module.exports.reset = (id, callback) => { }); }; +module.exports.activate = (id, callback) => { + module.exports.get(id, false, (err, campaign) => { + if (err) { + return callback(err); + } + + if (campaign.status !== 5) { + return callback(null, false); + } + + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + // campaigns marked as status=5 are paused + connection.query('UPDATE campaigns SET `status`=6, `status_change`=NOW() WHERE id=? LIMIT 1', [id], err => { + if (err) { + connection.release(); + return callback(err); + } + return callback(null, true); + }); + }); + }); +}; + +module.exports.inactivate = (id, callback) => { + module.exports.get(id, false, (err, campaign) => { + if (err) { + return callback(err); + } + + if (campaign.status !== 6) { + return callback(null, false); + } + + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + // campaigns marked as status=6 are paused + connection.query('UPDATE campaigns SET `status`=5, `status_change`=NOW() WHERE id=? LIMIT 1', [id], err => { + if (err) { + connection.release(); + return callback(err); + } + return callback(null, true); + }); + }); + }); +}; + module.exports.getMail = (campaignId, listId, subscriptionId, callback) => { campaignId = Number(campaignId) || 0; listId = Number(listId) || 0; diff --git a/meta.json b/meta.json index ee3d8093..827af726 100644 --- a/meta.json +++ b/meta.json @@ -1,3 +1,3 @@ { - "schemaVersion": 7 + "schemaVersion": 8 } diff --git a/package.json b/package.json index 9cab6f3f..8620b56c 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "hbs": "^4.0.0", "html-to-text": "^2.1.0", "humanize": "0.0.9", + "is-url": "^1.2.1", "isemail": "^2.1.0", "moment-timezone": "^0.5.3", "morgan": "^1.7.0", diff --git a/routes/campaigns.js b/routes/campaigns.js index 78e141c9..a2d7e704 100644 --- a/routes/campaigns.js +++ b/routes/campaigns.js @@ -196,7 +196,7 @@ router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) = }); router.post('/ajax', (req, res) => { - campaigns.filter(req.body, (err, data, total, filteredTotal) => { + campaigns.filter(req.body, Number(req.query.parent) || false, (err, data, total, filteredTotal) => { if (err) { return res.json({ error: err.message || err, @@ -217,6 +217,10 @@ router.post('/ajax', (req, res) => { return 'Finished'; case 4: return 'Paused'; + case 5: + return 'Inactive'; + case 6: + return 'Active'; } return 'Other'; }; @@ -256,6 +260,11 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => { campaign.isSending = campaign.status === 2; campaign.isFinished = campaign.status === 3; campaign.isPaused = campaign.status === 4; + campaign.isInactive = campaign.status === 5; + campaign.isActive = campaign.status === 6; + + campaign.isNormal = campaign.type === 1; + campaign.isRss = campaign.type === 2; campaign.isScheduled = campaign.scheduled && campaign.scheduled > new Date(); @@ -341,4 +350,32 @@ router.post('/pause', passport.parseForm, passport.csrfProtection, (req, res) => }); }); +router.post('/activate', passport.parseForm, passport.csrfProtection, (req, res) => { + campaigns.activate(req.body.id, (err, reset) => { + if (err) { + req.flash('danger', err && err.message || err); + } else if (reset) { + req.flash('success', 'Sending activated'); + } else { + req.flash('info', 'Could not activate sending'); + } + + return res.redirect('/campaigns/view/' + encodeURIComponent(req.body.id)); + }); +}); + +router.post('/inactivate', passport.parseForm, passport.csrfProtection, (req, res) => { + campaigns.inactivate(req.body.id, (err, reset) => { + if (err) { + req.flash('danger', err && err.message || err); + } else if (reset) { + req.flash('success', 'Sending paused'); + } else { + req.flash('info', 'Could not pause sending'); + } + + return res.redirect('/campaigns/view/' + encodeURIComponent(req.body.id)); + }); +}); + module.exports = router; diff --git a/setup/sql/upgrade-00008.sql b/setup/sql/upgrade-00008.sql new file mode 100644 index 00000000..90170ddb --- /dev/null +++ b/setup/sql/upgrade-00008.sql @@ -0,0 +1,24 @@ +# Header section +# Define incrementing schema version number +SET @schema_version = '8'; + +# Create new table to store RSS entries for RSS campaigns +CREATE TABLE `rss` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `parent` int(11) unsigned NOT NULL, + `guid` varchar(255) NOT NULL DEFAULT '', + `pubdate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `campaign` int(11) unsigned DEFAULT NULL, + `found` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `parent_2` (`parent`,`guid`), + KEY `parent` (`parent`), + CONSTRAINT `rss_ibfk_1` FOREIGN KEY (`parent`) REFERENCES `campaigns` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +ALTER TABLE `campaigns` ADD COLUMN `parent` int(11) unsigned DEFAULT NULL AFTER `type`; +CREATE INDEX parent_index ON `campaigns` (`parent`); + +# Footer section +LOCK TABLES `settings` WRITE; +INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version; +UNLOCK TABLES; diff --git a/views/campaigns/create-rss.hbs b/views/campaigns/create-rss.hbs index df6de899..a6e863a9 100644 --- a/views/campaigns/create-rss.hbs +++ b/views/campaigns/create-rss.hbs @@ -60,7 +60,7 @@
- + New entries from this RSS URL are sent out to list subscribers as email messages
diff --git a/views/campaigns/edit-rss.hbs b/views/campaigns/edit-rss.hbs index 68f7b4a5..253f57c2 100644 --- a/views/campaigns/edit-rss.hbs +++ b/views/campaigns/edit-rss.hbs @@ -71,7 +71,7 @@
- + New entries from this RSS URL are sent out to list subscribers as email messages
diff --git a/views/campaigns/view.hbs b/views/campaigns/view.hbs index 04c2b805..b81597ca 100644 --- a/views/campaigns/view.hbs +++ b/views/campaigns/view.hbs @@ -39,6 +39,11 @@ {{/if}} + {{#if isRss}} +
Feed URL
+
{{sourceUrl}}
+ {{/if}} +
Email "from name"
{{from}}
@@ -48,136 +53,200 @@
Email "subject line"
{{subject}}
- {{#unless isIdling}} -
Delivered
-
{{delivered}}
+ {{#if isNormal}} -
Bounced
-
{{bounced}}
+ {{#unless isIdling}} +
Delivered
+
{{delivered}}
-
Complaints
-
{{complained}}
+
Bounced
+
{{bounced}}
-
Unsubscribed
-
{{unsubscribed}}
+
Complaints
+
{{complained}}
-
Opened
-
-
-
- {{openRate}}% +
Unsubscribed
+
{{unsubscribed}}
+ +
Opened
+
+
+
+ {{openRate}}% +
- -
+ -
Clicked
-
-
-
- {{clicksRate}}% +
Clicked
+
+
+
+ {{clicksRate}}% +
- -
+ - {{/unless}} + {{/unless}} + {{/if}} -
-
- {{#if isIdling}} -
- - +{{#if isNormal}} -
-
-

Delay sending

-
-
-
- -
hours
+
+
+ {{#if isIdling}} + + + + +
+
+

Delay sending

+
+
+
+ +
hours
+
+
+
+
+ +
minutes
+
-
-
- -
minutes
-
+ + + + {{/if}} + + {{#if isSending}} + +
+ {{#if isScheduled}} +
+
+ + + + +
-
+

Sending scheduled {{scheduled}}

+ {{else}} +
+
+ + - -
- {{/if}} + + +
+

Sending…

+ {{/if}} + {{/if}} - {{#if isSending}} - -
- {{#if isScheduled}} + {{#if isPaused}}
-
+ -
-

Sending scheduled {{scheduled}}

+

Sending paused

+ {{/if}} + + {{#if isFinished}} +
+
+ + +
+ +
+ + +
+ + + + + +
+

All messages sent! Hit "Continue" if you you want to send this campaign to new subscribers

+ {{/if}} + +
+
+{{/if}} + +{{#if isRss}} + +
+
+ {{#if isActive}} +
+
+ + +
+ + +
+ + Campaign status: ACTIVE {{else}}
-
+ - -
-
-

Sending…

- {{/if}} - {{/if}} - {{#if isPaused}} -
-
- - - - -
-
-

Sending paused

- {{/if}} - - {{#if isFinished}} -
-
- - -
- -
- - -
- - - - - -
-

All messages sent! Hit "Continue" if you you want to send this campaign to new subscribers

- {{/if}} +
+ Campaign status: INACTIVE + {{/if}} +
-
+ +
+ + + + + + + + + +
+ # + + Name + + Description + + Status + + Created + +   +
+
+{{/if}}