diff --git a/index.js b/index.js index fad50a75..d3e1e036 100644 --- a/index.js +++ b/index.js @@ -9,9 +9,9 @@ let log = require('npmlog'); let app = require('./app'); let http = require('http'); let sender = require('./services/sender'); -let importer = require('./services/importer'); // eslint-disable-line global-require -let verpServer = require('./services/verp-server'); // eslint-disable-line global-require -let testServer = require('./services/test-server'); // eslint-disable-line global-require +let importer = require('./services/importer'); +let verpServer = require('./services/verp-server'); +let testServer = require('./services/test-server'); let dbcheck = require('./lib/dbcheck'); let port = config.www.port; diff --git a/lib/dbcheck.js b/lib/dbcheck.js index 47884ad1..45d2be66 100644 --- a/lib/dbcheck.js +++ b/lib/dbcheck.js @@ -1,5 +1,9 @@ 'use strict'; +/* +This module handles Mailtrain database initialization and upgrades +*/ + let config = require('config'); let mysql = require('mysql'); let log = require('npmlog'); diff --git a/lib/models/subscriptions.js b/lib/models/subscriptions.js index 9bb6ef3c..9d33df52 100644 --- a/lib/models/subscriptions.js +++ b/lib/models/subscriptions.js @@ -317,7 +317,10 @@ module.exports.insert = (listId, meta, subscription, callback) => { }); } connection.release(); - return callback(null, entryId); + return callback(null, { + entryId, + inserted: !existing + }); }); }); } else { @@ -329,7 +332,10 @@ module.exports.insert = (listId, meta, subscription, callback) => { }); } connection.release(); - return callback(null, entryId); + return callback(null, { + entryId, + inserted: !existing + }); }); } }); @@ -692,7 +698,7 @@ module.exports.updateImport = (listId, importId, data, callback) => { let keys = []; let values = []; - let allowedKeys = ['type', 'path', 'size', 'delimiter', 'status', 'error', 'processed', 'mapping', 'finished']; + let allowedKeys = ['type', 'path', 'size', 'delimiter', 'status', 'error', 'processed', 'new', 'failed', 'mapping', 'finished']; Object.keys(data).forEach(key => { let value = data[key]; key = tools.toDbKey(key); @@ -708,11 +714,25 @@ module.exports.updateImport = (listId, importId, data, callback) => { } let query = 'UPDATE importer SET ' + keys.map(key => '`' + key + '`=?') + ' WHERE id=? AND list=? LIMIT 1'; connection.query(query, values.concat([importId, listId]), (err, result) => { - connection.release(); if (err) { + connection.release(); return callback(err); } - return callback(null, result && result.affectedRows || false); + + let affected = result && result.affectedRows || false; + + if (data.failed === 0) { + // remove entries from import_failed table + let query = 'DELETE FROM `import_failed` WHERE `import`=?'; + connection.query(query, [importId], () => { + connection.release(); + return callback(null, affected); + }); + return; + } + + connection.release(); + return callback(null, affected); }); }); }; @@ -758,6 +778,29 @@ module.exports.getImport = (listId, importId, callback) => { }); }; +module.exports.getFailedImports = (importId, callback) => { + importId = Number(importId) || 0; + + if (importId < 1) { + return callback(new Error('Missing Import ID')); + } + + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + let query = 'SELECT * FROM import_failed WHERE import=? LIMIT 1000'; + connection.query(query, [importId], (err, rows) => { + connection.release(); + if (err) { + return callback(err); + } + + return callback(null, (rows || []).map(tools.convertKeys)); + }); + }); +}; + module.exports.listImports = (listId, callback) => { listId = Number(listId) || 0; diff --git a/meta.json b/meta.json index 604494a3..5d571180 100644 --- a/meta.json +++ b/meta.json @@ -1,3 +1,3 @@ { - "schemaVersion": 1 + "schemaVersion": 2 } diff --git a/nitrous-post-create.sh b/nitrous-post-create.sh index 93d8cd9e..47841ed5 100755 --- a/nitrous-post-create.sh +++ b/nitrous-post-create.sh @@ -13,7 +13,6 @@ MYSQL_PASSWORD=`pwgen -1` mysql -u root -e "CREATE USER 'mailtrain'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';" mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'%' WITH GRANT OPTION;" mysql -u mailtrain --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain;" -mysql -u mailtrain --password="$MYSQL_PASSWORD" -D mailtrain < setup/mailtrain.sql cat >> config/production.toml < { } entry.created = entry.created && entry.created.toISOString(); entry.finished = entry.finished && entry.finished.toISOString(); + entry.updated = entry.processed - entry.new; return entry; }); list.csrfToken = req.csrfToken(); @@ -314,13 +315,13 @@ router.get('/subscription/:id/edit/:cid', passport.csrfProtection, (req, res) => }); router.post('/subscription/add', passport.parseForm, passport.csrfProtection, (req, res) => { - subscriptions.insert(req.body.list, false, req.body, (err, entryId) => { + subscriptions.insert(req.body.list, false, req.body, (err, response) => { if (err) { req.flash('danger', err && err.message || err || 'Could not add subscription'); return res.redirect('/lists/subscription/' + encodeURIComponent(req.body.list) + '/add?' + tools.queryParams(req.body)); } - if (entryId) { + if (response.entryId) { req.flash('success', req.body.email + ' was successfully added to your list'); } else { req.flash('warning', req.body.email + ' was not added to your list'); @@ -429,7 +430,7 @@ router.get('/subscription/:id/import/:importId', passport.csrfProtection, (req, } subscriptions.getImport(req.params.id, req.params.importId, (err, data) => { - if (err || !list) { + if (err || !data) { req.flash('danger', err && err.message || err || 'Could not find import data with specified ID'); return res.redirect('/lists'); } @@ -597,7 +598,9 @@ router.post('/subscription/import-restart', passport.parseForm, passport.csrfPro status: 1, error: null, finished: null, - processed: 0 + processed: 0, + new: 0, + failed: 0 }, (err, importer) => { if (err || !importer) { req.flash('danger', err && err.message || err || 'Could not find import data with specified ID'); @@ -610,4 +613,35 @@ router.post('/subscription/import-restart', passport.parseForm, passport.csrfPro }); }); +router.get('/subscription/:id/import/:importId/failed', (req, res) => { + let start = 0; + lists.get(req.params.id, (err, list) => { + if (err || !list) { + req.flash('danger', err && err.message || err || 'Could not find list with specified ID'); + return res.redirect('/lists'); + } + + subscriptions.getImport(req.params.id, req.params.importId, (err, data) => { + if (err || !data) { + req.flash('danger', err && err.message || err || 'Could not find import data with specified ID'); + return res.redirect('/lists'); + } + subscriptions.getFailedImports(req.params.importId, (err, rows) => { + if (err) { + req.flash('danger', err && err.message || err); + return res.redirect('/lists'); + } + + data.rows = rows.map((row, i) => { + row.index = start + i + 1; + return row; + }); + data.list = list; + + res.render('lists/subscription/import-failed', data); + }); + }); + }); +}); + module.exports = router; diff --git a/services/importer.js b/services/importer.js index 7f18d82a..2095b5cf 100644 --- a/services/importer.js +++ b/services/importer.js @@ -139,18 +139,38 @@ function processImport(data, callback) { tools.validateEmail(entry.email, true, err => { if (err) { - log.verbose('Import', 'Failed processing row %s: %s', entry.email, err.message); - return setImmediate(processRows); + let reason = (err.message || '').toString().trim().replace(/^[a-z]Error:\s*/i, ''); + log.verbose('Import', 'Failed processing row %s: %s', entry.email, reason); + db.getConnection((err, connection) => { + if (err) { + log.error('Import', err.stack); + return setImmediate(processRows); + } + + let query = 'INSERT INTO import_failed (`import`, `email`, `reason`) VALUES(?,?,?)'; + connection.query(query, [data.id, entry.email, reason], err => { + if (err) { + connection.release(); + return setImmediate(processRows); + } + let query = 'UPDATE importer SET `failed`=`failed`+1 WHERE `id`=? LIMIT 1'; + connection.query(query, [data.id], () => { + connection.release(); + return setImmediate(processRows); + }); + }); + }); + return; } subscriptions.insert(listId, { imported: data.id, status: data.type - }, entry, (err, entryId) => { + }, entry, (err, response) => { if (err) { // ignore log.error('Import', err.stack); - } else if (entryId) { + } else if (response.entryId) { //log.verbose('Import', 'Inserted %s as %s', entry.email, entryId); } @@ -160,7 +180,15 @@ function processImport(data, callback) { return setImmediate(processRows); } - let query = 'UPDATE importer SET `processed`=`processed`+1 WHERE `id`=? LIMIT 1'; + let query; + if (response.inserted) { + // this record did not exist before, count as new + query = 'UPDATE importer SET `processed`=`processed`+1, `new`=`new`+1 WHERE `id`=? LIMIT 1'; + } else { + // it's an existing record + query = 'UPDATE importer SET `processed`=`processed`+1 WHERE `id`=? LIMIT 1'; + } + connection.query(query, [data.id], () => { connection.release(); return setImmediate(processRows); diff --git a/setup/sql/mailtrain.sql b/setup/sql/mailtrain.sql index 2e61c3f0..c49a62db 100644 --- a/setup/sql/mailtrain.sql +++ b/setup/sql/mailtrain.sql @@ -157,7 +157,7 @@ CREATE TABLE `segments` ( `created` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `list` (`list`), - KEY `name` (`name`(191)), + KEY `name` (`name`), CONSTRAINT `segments_ibfk_1` FOREIGN KEY (`list`) REFERENCES `lists` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; @@ -222,7 +222,7 @@ CREATE TABLE `users` ( ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4; LOCK TABLES `users` WRITE; -INSERT INTO `users` VALUES (1,'admin','$2a$10$mzKU71G62evnGB2PvQA4k..Wf9jASk.c7a8zRMHh6qQVjYJ2r/g/K','admin@example.com',NULL,NULL,'2016-04-20 17:20:48'); +INSERT INTO `users` VALUES (1,'admin','$2a$10$mzKU71G62evnGB2PvQA4k..Wf9jASk.c7a8zRMHh6qQVjYJ2r/g/K','admin@example.com',NULL,NULL,NOW()); UNLOCK TABLES; SET UNIQUE_CHECKS=1; diff --git a/setup/sql/upgrade-00002.sql b/setup/sql/upgrade-00002.sql new file mode 100644 index 00000000..73c4769c --- /dev/null +++ b/setup/sql/upgrade-00002.sql @@ -0,0 +1,12 @@ +# Header section +# Define incrementing schema version number +SET @schema_version = '2'; + +# Adds new column 'failed' to importer table. Includes the count of failed addresses for an import +ALTER TABLE importer ADD COLUMN `failed` INT(11) UNSIGNED NOT NULL DEFAULT '0' AFTER `processed`; +ALTER TABLE importer ADD COLUMN `new` INT(11) UNSIGNED NOT NULL DEFAULT '0' AFTER `processed`; + +# Footer section +LOCK TABLES `settings` WRITE; +INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version; +UNLOCK TABLES; diff --git a/views/lists/subscription/import-failed.hbs b/views/lists/subscription/import-failed.hbs new file mode 100644 index 00000000..18609659 --- /dev/null +++ b/views/lists/subscription/import-failed.hbs @@ -0,0 +1,44 @@ +
    +
  1. Home
    2. +
  2. Lists
    3. +
  3. {{list.name}}
    4. +
  4. Import status
    5. +
+ +

{{list.name}} Failed addresses Back to list

+ +
+ +
+ + + + + + + {{#if rows}} + + {{#each rows}} + + + + + + {{/each}} + + {{/if}} +
+ # + + Address + + Fail reason +
+ {{index}} + + {{email}} + + + {{reason}} +
+
diff --git a/views/lists/view.hbs b/views/lists/view.hbs index 701108ea..a5eed89b 100644 --- a/views/lists/view.hbs +++ b/views/lists/view.hbs @@ -121,7 +121,13 @@ Type - Processed + Added + + + Updated + + + Failed Status @@ -154,7 +160,13 @@ {{importType}} - {{processed}} + {{new}} + + + {{updated}} + + + {{#if failed}}{{failed}}{{else}} 0 {{/if}} {{#if error}} @@ -175,7 +187,7 @@ {{/each}} {{else}} - + No data available in table