From c9136f8abd1c4c1eb08615fcbd47ea6d1d9c51fb Mon Sep 17 00:00:00 2001 From: vladimir Date: Mon, 10 Apr 2017 19:09:40 +0200 Subject: [PATCH] Add blacklist function --- app.js | 2 + lib/models/blacklist.js | 86 ++++++++++++++++++++++++++++ meta.json | 2 +- routes/api.js | 80 ++++++++++++++++++++++++++ routes/blacklist.js | 68 ++++++++++++++++++++++ routes/campaigns.js | 3 + services/sender.js | 94 +++++++++++++++++++++---------- setup/sql/upgrade-00025.sql | 17 ++++++ views/blacklist.hbs | 38 +++++++++++++ views/campaigns/blacklisted.hbs | 54 ++++++++++++++++++ views/campaigns/view.hbs | 7 +++ views/layout.hbs | 5 ++ views/lists/subscription/edit.hbs | 7 +++ views/users/api.hbs | 71 +++++++++++++++++------ 14 files changed, 486 insertions(+), 48 deletions(-) create mode 100644 lib/models/blacklist.js create mode 100644 routes/blacklist.js create mode 100644 setup/sql/upgrade-00025.sql create mode 100644 views/blacklist.hbs create mode 100644 views/campaigns/blacklisted.hbs diff --git a/app.js b/app.js index ba4e27bd..ab7880f2 100644 --- a/app.js +++ b/app.js @@ -36,6 +36,7 @@ let webhooks = require('./routes/webhooks'); let subscription = require('./routes/subscription'); let archive = require('./routes/archive'); let api = require('./routes/api'); +let blacklist = require('./routes/blacklist'); let editorapi = require('./routes/editorapi'); let grapejs = require('./routes/grapejs'); let mosaico = require('./routes/mosaico'); @@ -207,6 +208,7 @@ app.use('/lists', lists); app.use('/templates', templates); app.use('/campaigns', campaigns); app.use('/settings', settings); +app.use('/blacklist', blacklist); app.use('/links', links); app.use('/fields', fields); app.use('/forms', forms); diff --git a/lib/models/blacklist.js b/lib/models/blacklist.js new file mode 100644 index 00000000..5aae8ccc --- /dev/null +++ b/lib/models/blacklist.js @@ -0,0 +1,86 @@ +'use strict'; + +let db = require('../db'); + +module.exports.get = (start, limit, search, callback) => { + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + search = '%' + search + '%'; + connection.query('SELECT SQL_CALC_FOUND_ROWS `email` FROM blacklist WHERE `email` LIKE ? ORDER BY `email` LIMIT ? OFFSET ?', [search, limit, start], (err, rows) => { + if (err) { + return callback(err); + } + + connection.query('SELECT FOUND_ROWS() AS total', (err, total) => { + connection.release(); + if (err) { + return callback(err); + } + let emails = []; + rows.forEach(email => { + emails.push(email.email); + }); + return callback(null, emails, total && total[0] && total[0].total); + }); + }); + }); +}; + +module.exports.add = (email, callback) => { + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + connection.query('INSERT IGNORE INTO `blacklist` (`email`) VALUES(?)', email, err => { + if (err) { + return callback(err); + } + + connection.release(); + return callback(null, null); + + }); + }); +}; + +module.exports.delete = (email, callback) => { + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + connection.query('DELETE FROM `blacklist` WHERE `email`=?', email, err => { + if (err) { + return callback(err); + } + + connection.release(); + return callback(null, null); + + }); + }); +}; + +module.exports.isblacklisted = (email, callback) => { + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + connection.query('SELECT `email` FROM blacklist WHERE `email`=?', email, (err, rows) => { + if (err) { + return callback(err); + } + + connection.release(); + if (rows.length > 0) { + return callback(null, true); + } else { + return callback(null, false); + } + }); + }); +}; diff --git a/meta.json b/meta.json index 58d89ef6..1302fec6 100644 --- a/meta.json +++ b/meta.json @@ -1,3 +1,3 @@ { - "schemaVersion": 24 + "schemaVersion": 25 } diff --git a/routes/api.js b/routes/api.js index e1d3943e..ce71a7c1 100644 --- a/routes/api.js +++ b/routes/api.js @@ -3,6 +3,7 @@ let users = require('../lib/models/users'); let lists = require('../lib/models/lists'); let fields = require('../lib/models/fields'); +let blacklist = require('../lib/models/blacklist'); let subscriptions = require('../lib/models/subscriptions'); let tools = require('../lib/tools'); let express = require('express'); @@ -326,4 +327,83 @@ router.post('/field/:listId', (req, res) => { }); }); +router.post('/blacklist/add', (req, res) => { + let input = {}; + Object.keys(req.body).forEach(key => { + input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim(); + }); + if (!(input.EMAIL) || (input.EMAIL === '')) { + res.status(500); + return res.json({ + error: 'EMAIL argument are required', + data: [] + }); + } + blacklist.add(input.EMAIL, (err) =>{ + if (err) { + res.status(500); + return res.json({ + error: err.message || err, + data: [] + }); + } + res.status(200); + res.json({ + data: [] + }); + }); +}); + +router.post('/blacklist/delete', (req, res) => { + let input = {}; + Object.keys(req.body).forEach(key => { + input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim(); + }); + if (!(input.EMAIL) || (input.EMAIL === '')) { + res.status(500); + return res.json({ + error: 'EMAIL argument are required', + data: [] + }); + } + blacklist.delete(input.EMAIL, (err) =>{ + if (err) { + res.status(500); + return res.json({ + error: err.message || err, + data: [] + }); + } + res.status(200); + res.json({ + data: [] + }); + }); +}); + +router.get('/blacklist/get', (req, res) => { + let start = parseInt(req.query.start || 0, 10); + let limit = parseInt(req.query.limit || 10000, 10); + let search = req.query.search || ''; + + blacklist.get(start, limit, search, (err, data, total) => { + if (err) { + res.status(500); + return res.json({ + error: err.message || err, + data: [] + }); + } + res.status(200); + res.json({ + data: { + total: total, + start: start, + limit: limit, + emails: data + } + }); + }); +}); + module.exports = router; diff --git a/routes/blacklist.js b/routes/blacklist.js new file mode 100644 index 00000000..2f009267 --- /dev/null +++ b/routes/blacklist.js @@ -0,0 +1,68 @@ +'use strict'; +let express = require('express'); +let router = new express.Router(); +let passport = require('../lib/passport'); +let htmlescape = require('escape-html'); +let blacklist = require('../lib/models/blacklist'); +let tools = require('../lib/tools'); +let helpers = require('../lib/helpers'); +let _ = require('../lib/translate')._; + +router.all('/*', (req, res, next) => { + if (!req.user) { + req.flash('danger', _('Need to be logged in to access restricted content')); + return res.redirect('/users/login?next=' + encodeURIComponent(req.originalUrl)); + } + res.setSelectedMenu('blacklist'); + next(); +}); + +router.get('/', passport.csrfProtection, (req, res) => { + res.render('blacklist', {csrfToken: req.csrfToken()}); +}); + +router.post('/ajax/', (req, res) => { + let start = parseInt(req.body.start || 0, 10); + let limit = parseInt(req.body.length || 50, 10); + let search = req.body.search.value || ''; + blacklist.get(start, limit, search, (err, data, total) => { + if (err) { + req.flash('danger', err.message || err); + return res.redirect('/'); + } + res.json({ + draw: req.body.draw, + recordsTotal: total, + recordsFiltered: total, + data: data.map((row, i) => [ + (Number(req.body.start) || 0) + 1 + i, + htmlescape(row), + '' + ]) + }); + }); +}); + +router.post('/ajax/add', passport.csrfProtection, (req, res) => { + let email = req.body.email; + blacklist.add(email, (err) => { + if (err) { + req.flash('danger', err.message || err); + return res.redirect(req.body.next); + } + return res.redirect(req.body.next) + }); +}); + +router.post('/ajax/delete', passport.csrfProtection, (req, res) => { + let email = req.body.email; + blacklist.delete(email, (err) => { + if (err) { + req.flash('danger', err.message || err); + return res.redirect(req.body.next); + } + return res.redirect(req.body.next); + }); +}); + +module.exports = router; diff --git a/routes/campaigns.js b/routes/campaigns.js index 81dcd2ba..08b3e230 100644 --- a/routes/campaigns.js +++ b/routes/campaigns.js @@ -425,6 +425,9 @@ router.get('/status/:id/:status', passport.csrfProtection, (req, res) => { case 'complained': status = 4; break; + case 'blacklisted': + status = 5; + break; default: req.flash('danger', _('Unknown status selector')); return res.redirect('/campaigns'); diff --git a/services/sender.js b/services/sender.js index ab84485c..138850c9 100644 --- a/services/sender.js +++ b/services/sender.js @@ -8,6 +8,7 @@ let mailer = require('../lib/mailer'); let campaigns = require('../lib/models/campaigns'); let segments = require('../lib/models/segments'); let lists = require('../lib/models/lists'); +let blacklist = require('../lib/models/blacklist'); let fields = require('../lib/models/fields'); let settings = require('../lib/models/settings'); let links = require('../lib/models/links'); @@ -491,53 +492,86 @@ let sendLoop = () => { return; } - let tryCount = 0; - let trySend = () => { - tryCount++; + blacklist.isblacklisted(mail.to.address, (err, blacklisted) => { + if (err) { + log.error('Mail', err); + setTimeout(getNext, mailing_timeout); + return; + } + if (!blacklisted) { + let tryCount = 0; + let trySend = () => { + tryCount++; - // send the message - mailer.transport.sendMail(mail, (err, info) => { - if (err) { - log.error('Mail', err.stack); - if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) { - // temporary error, try again - return setTimeout(trySend, tryCount * 1000); - } - } - - let status = err ? 2 : 1; - let response = err && (err.response || err.message) || info.response || info.messageId; - let responseId = response.split(/\s+/).pop(); - - db.getConnection((err, connection) => { + // send the message + mailer.transport.sendMail(mail, (err, info) => { if (err) { log.error('Mail', err.stack); - return; + if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) { + // temporary error, try again + return setTimeout(trySend, tryCount * 1000); + } } - let query = 'UPDATE `campaigns` SET `delivered`=`delivered`+1 ' + (status === 2 ? ', `bounced`=`bounced`+1 ' : '') + ' WHERE id=? LIMIT 1'; + let status = err ? 2 : 1; + let response = err && (err.response || err.message) || info.response || info.messageId; + let responseId = response.split(/\s+/).pop(); - connection.query(query, [message.campaignId], err => { + db.getConnection((err, connection) => { if (err) { log.error('Mail', err.stack); + return; } - let query = 'UPDATE `campaign__' + message.campaignId + '` SET status=?, response=?, response_id=?, updated=NOW() WHERE id=? LIMIT 1'; + let query = 'UPDATE `campaigns` SET `delivered`=`delivered`+1 ' + (status === 2 ? ', `bounced`=`bounced`+1 ' : '') + ' WHERE id=? LIMIT 1'; - connection.query(query, [status, response, responseId, message.id], err => { - connection.release(); + connection.query(query, [message.campaignId], err => { if (err) { log.error('Mail', err.stack); - } else { - // log.verbose('Mail', 'Message sent and status updated for %s', message.subscription.cid); } + + let query = 'UPDATE `campaign__' + message.campaignId + '` SET status=?, response=?, response_id=?, updated=NOW() WHERE id=? LIMIT 1'; + + connection.query(query, [status, response, responseId, message.id], err => { + connection.release(); + if (err) { + log.error('Mail', err.stack); + } else { + // log.verbose('Mail', 'Message sent and status updated for %s', message.subscription.cid); + } + }); }); }); }); - }); - }; - setImmediate(trySend); - setImmediate(() => mailer.transport.checkThrottling(getNext)); + }; + setImmediate(trySend); + setImmediate(() => mailer.transport.checkThrottling(getNext)); + } else { + db.getConnection((err, connection) => { + if (err) { + log.error('Mail', err); + return; + } + + let query = 'UPDATE `campaigns` SET `blacklisted`=`blacklisted`+1 WHERE id=? LIMIT 1'; + + connection.query(query, [message.campaignId], err => { + if (err) { + log.error('Mail', err); + } + + let query = 'UPDATE `campaign__' + message.campaignId + '` SET status=?, response=?, response_id=?, updated=NOW() WHERE id=? LIMIT 1'; + + connection.query(query, [5, 'blacklisted', 'blacklisted', message.id], err => { + connection.release(); + if (err) { + log.error('Mail', err); + } + }); + }); + }); + } + }); }); }); }; diff --git a/setup/sql/upgrade-00025.sql b/setup/sql/upgrade-00025.sql new file mode 100644 index 00000000..9b3fba54 --- /dev/null +++ b/setup/sql/upgrade-00025.sql @@ -0,0 +1,17 @@ +# Header section +# Define incrementing schema version number +SET @schema_version = '25'; + +# Create table to store global blacklist +CREATE TABLE `blacklist` ( + `email` varchar(255) NOT NULL, + PRIMARY KEY (`email`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +#Alter table campaigns +ALTER TABLE `campaigns` ADD COLUMN `blacklisted` int(11) unsigned NOT NULL DEFAULT '0' AFTER `delivered`; + +# Footer section +LOCK TABLES `settings` WRITE; +INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version; +UNLOCK TABLES; diff --git a/views/blacklist.hbs b/views/blacklist.hbs new file mode 100644 index 00000000..4a1fddca --- /dev/null +++ b/views/blacklist.hbs @@ -0,0 +1,38 @@ +
    +
  1. {{#translate}}Home{{/translate}}
    2. +
  2. {{#translate}}Blacklist{{/translate}}
    3. +
+ +

{{#translate}}Blacklist{{/translate}}

+ +
+ + +
+ +
+ +
+ +
+ + + +
+ +
+
+ + + + + + +
+ # + + {{#translate}}Email{{/translate}} + +   +
+
diff --git a/views/campaigns/blacklisted.hbs b/views/campaigns/blacklisted.hbs new file mode 100644 index 00000000..f34d775a --- /dev/null +++ b/views/campaigns/blacklisted.hbs @@ -0,0 +1,54 @@ +
    +
  1. {{#translate}}Home{{/translate}}
    2. +
  2. {{#translate}}Campaigns{{/translate}}
    3. + {{#if parent}} +
  3. {{parent.name}}
    4. + {{/if}} +
  4. {{name}}
    5. +
  5. {{#translate}}Blacklisted info{{/translate}}
    6. +
+ +

{{name}} {{#translate}}Blacklisted info{{/translate}} {{#translate}}View campaign{{/translate}}

+ +
+ +{{#if description}} +
{{{description}}}
+{{/if}} + +
+ +
+ +
{{#translate}}Subscribers who blacklisted by global blacklist:{{/translate}}
+
+
+ + + + + + + + + + + + +
+ # + + {{#translate}}Address{{/translate}} + + {{#translate}}First Name{{/translate}} + + {{#translate}}Last Name{{/translate}} + + {{#translate}}Reason{{/translate}} + + {{#translate}}Time{{/translate}} +
+
+
+
+
diff --git a/views/campaigns/view.hbs b/views/campaigns/view.hbs index 4528780b..a8e981b3 100644 --- a/views/campaigns/view.hbs +++ b/views/campaigns/view.hbs @@ -130,6 +130,13 @@ +
{{#translate}}Blacklisted{{/translate}}
+
+
+ {{blacklisted}} +
+
+
{{#translate}}Bounced{{/translate}}
diff --git a/views/layout.hbs b/views/layout.hbs index 541d089a..339e8006 100644 --- a/views/layout.hbs +++ b/views/layout.hbs @@ -82,6 +82,11 @@ {{#translate}}Settings{{/translate}} +
  • + + {{#translate}}Blacklist{{/translate}} + +
  • {{#translate}}API{{/translate}} diff --git a/views/lists/subscription/edit.hbs b/views/lists/subscription/edit.hbs index 72227d5c..0f4a8b2c 100644 --- a/views/lists/subscription/edit.hbs +++ b/views/lists/subscription/edit.hbs @@ -17,6 +17,12 @@ +
    + + + +
    +
    @@ -166,6 +172,7 @@
    + {{#if isSubscribed}} {{/if}} diff --git a/views/users/api.hbs b/views/users/api.hbs index 3aef1641..1a322032 100644 --- a/views/users/api.hbs +++ b/views/users/api.hbs @@ -142,10 +142,32 @@ 
    curl -XPOST {{serviceUrl}}api/delete/B16uVTdW?access_token={{accessToken}} \
 --data 'EMAIL=test@example.com'
    -

    POST /api/field/:listId – {{#translate}}Add new custom field{{/translate}}

    +

    GET /api/blacklist/get – {{#translate}}Get list of blacklisted emails{{/translate}}

    - {{#translate}}This API call creates a new custom field for a list.{{/translate}} + {{#translate}}This API call get list of blacklisted emails.{{/translate}} +

    + +

    + GET {{#translate}}arguments{{/translate}} +

    +
      +
    • access_token – {{#translate}}your personal access token{{/translate}} +
    • start – {{#translate}}Start position{{/translate}} ({{#translate}}optional, default 0{{/translate}})
      • +
    • limit – {{#translate}}limit emails count in response{{/translate}} ({{#translate}}optional, default 10000{{/translate}})
      • +
    • search – {{#translate}}filter by part of email{{/translate}} ({{#translate}}optional, default ''{{/translate}})
      • +
    + +

    + {{#translate}}Example{{/translate}} +

    + +
    curl -XGET '{{serviceUrl}}api/blacklist/get?access_token={{accessToken}}&limit=10&start=10&search=gmail'
    + +

    POST /api/blacklist/add – {{#translate}}Add email to blacklist{{/translate}}

    + +

    + {{#translate}}This API call either add emails to blacklist{{/translate}}

    @@ -159,24 +181,39 @@ POST {{#translate}}arguments{{/translate}}

      -
    • NAME – {{#translate}}field name{{/translate}} ({{#translate}}required{{/translate}})
      • -
    • TYPE – {{#translate}}one of the following types:{{/translate}} -
        - {{#each allowedTypes}} -
      • - {{type}} {{description}} -
        • - {{/each}} -
      -
      • -
    • GROUP – {{#translate}}If the type is 'option' then you also need to specify the parent element ID{{/translate}}
      • -
    • GROUP_TEMPLATE – {{#translate}}Template for the group element. If not set, then values of the elements are joined with commas{{/translate}}
      • -
    • VISIBLE – yes/no, {{#translate}}if not visible then the subscriber can not view or modify this value at the profile page{{/translate}}
      • +
    • EMAIL – {{#translate}}email address{{/translate}} ({{#translate}}required{{/translate}})

    {{#translate}}Example{{/translate}}

    -
    curl -XPOST {{serviceUrl}}api/field/B16uVTdW?access_token={{accessToken}} \
---data 'NAME=Birthday&TYPE=birthday-us&VISIBLE=yes'
    +
    curl -XPOST '{{serviceUrl}}api/blacklist/add?access_token={{accessToken}}' \
+--data 'EMAIL=test@example.com&'
    + +

    POST /api/blacklist/delete – {{#translate}}Delete email from blacklist{{/translate}}

    + +

    + {{#translate}}This API call either delete emails from blacklist{{/translate}} +

    + +

    + GET {{#translate}}arguments{{/translate}} +

    +
      +
    • access_token – {{#translate}}your personal access token{{/translate}} +
    + +

    + POST {{#translate}}arguments{{/translate}} +

    +
      +
    • EMAIL – {{#translate}}email address{{/translate}} ({{#translate}}required{{/translate}})
      • +
    + +

    + {{#translate}}Example{{/translate}} +

    + +
    curl -XPOST '{{serviceUrl}}api/blacklist/delete?access_token={{accessToken}}' \
+--data 'EMAIL=test@example.com&'