public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Display failed imported addresses

Browse source
This commit is contained in:
Andris Reinman 2016-04-25 15:39:17 +03:00
parent 873d88658c
commit 172c8ce56f
12 changed files with 201 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

6
index.js
View file

@ -9,9 +9,9 @@ let log = require('npmlog');
let app = require('./app');
let http = require('http');
let sender = require('./services/sender');
let importer = require('./services/importer'); // eslint-disable-line global-require
let verpServer = require('./services/verp-server'); // eslint-disable-line global-require
let testServer = require('./services/test-server'); // eslint-disable-line global-require
let importer = require('./services/importer');
let verpServer = require('./services/verp-server');
let testServer = require('./services/test-server');
let dbcheck = require('./lib/dbcheck');
let port = config.www.port;

4
lib/dbcheck.js
View file

@ -1,5 +1,9 @@
'use strict';
/*
This module handles Mailtrain database initialization and upgrades
*/
let config = require('config');
let mysql = require('mysql');
let log = require('npmlog');

53
lib/models/subscriptions.js
View file

@ -317,7 +317,10 @@ module.exports.insert = (listId, meta, subscription, callback) => {
});
}
connection.release();
return callback(null, entryId);
return callback(null, {
entryId,
inserted: !existing
});
});
});
} else {
@ -329,7 +332,10 @@ module.exports.insert = (listId, meta, subscription, callback) => {
});
}
connection.release();
return callback(null, entryId);
return callback(null, {
entryId,
inserted: !existing
});
});
}
});
@ -692,7 +698,7 @@ module.exports.updateImport = (listId, importId, data, callback) => {
let keys = [];
let values = [];
let allowedKeys = ['type', 'path', 'size', 'delimiter', 'status', 'error', 'processed', 'mapping', 'finished'];
let allowedKeys = ['type', 'path', 'size', 'delimiter', 'status', 'error', 'processed', 'new', 'failed', 'mapping', 'finished'];
Object.keys(data).forEach(key => {
let value = data[key];
key = tools.toDbKey(key);
@ -708,11 +714,25 @@ module.exports.updateImport = (listId, importId, data, callback) => {
}
let query = 'UPDATE importer SET ' + keys.map(key => '`' + key + '`=?') + ' WHERE id=? AND list=? LIMIT 1';
connection.query(query, values.concat([importId, listId]), (err, result) => {
connection.release();
if (err) {
connection.release();
return callback(err);
}
return callback(null, result && result.affectedRows || false);
let affected = result && result.affectedRows || false;
if (data.failed === 0) {
// remove entries from import_failed table
let query = 'DELETE FROM `import_failed` WHERE `import`=?';
connection.query(query, [importId], () => {
connection.release();
return callback(null, affected);
});
return;
}
connection.release();
return callback(null, affected);
});
});
};
@ -758,6 +778,29 @@ module.exports.getImport = (listId, importId, callback) => {
});
};
module.exports.getFailedImports = (importId, callback) => {
importId = Number(importId) || 0;
if (importId < 1) {
return callback(new Error('Missing Import ID'));
}
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
let query = 'SELECT * FROM import_failed WHERE import=? LIMIT 1000';
connection.query(query, [importId], (err, rows) => {
connection.release();
if (err) {
return callback(err);
}
return callback(null, (rows || []).map(tools.convertKeys));
});
});
};
module.exports.listImports = (listId, callback) => {
listId = Number(listId) || 0;

2
meta.json
View file

@ -1,3 +1,3 @@
{
"schemaVersion": 1
"schemaVersion": 2
}

1
nitrous-post-create.sh
View file

@ -13,7 +13,6 @@ MYSQL_PASSWORD=`pwgen -1`
mysql -u root -e "CREATE USER 'mailtrain'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';"
mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'%' WITH GRANT OPTION;"
mysql -u mailtrain --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain;"
mysql -u mailtrain --password="$MYSQL_PASSWORD" -D mailtrain < setup/mailtrain.sql
cat >> config/production.toml <<EOT
[log]

2
package.json
View file

@ -52,7 +52,7 @@
"nodemailer": "^2.3.2",
"nodemailer-openpgp": "^1.0.2",
"npmlog": "^2.0.3",
"openpgp": "^2.2.1",
"openpgp": "^2.2.2",
"passport": "^0.3.2",
"passport-local": "^1.0.0",
"request": "^2.72.0",

42
routes/lists.js
View file

@ -232,6 +232,7 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
}
entry.created = entry.created && entry.created.toISOString();
entry.finished = entry.finished && entry.finished.toISOString();
entry.updated = entry.processed - entry.new;
return entry;
});
list.csrfToken = req.csrfToken();
@ -314,13 +315,13 @@ router.get('/subscription/:id/edit/:cid', passport.csrfProtection, (req, res) =>
});
router.post('/subscription/add', passport.parseForm, passport.csrfProtection, (req, res) => {
subscriptions.insert(req.body.list, false, req.body, (err, entryId) => {
subscriptions.insert(req.body.list, false, req.body, (err, response) => {
if (err) {
req.flash('danger', err && err.message || err || 'Could not add subscription');
return res.redirect('/lists/subscription/' + encodeURIComponent(req.body.list) + '/add?' + tools.queryParams(req.body));
}
if (entryId) {
if (response.entryId) {
req.flash('success', req.body.email + ' was successfully added to your list');
} else {
req.flash('warning', req.body.email + ' was not added to your list');
@ -429,7 +430,7 @@ router.get('/subscription/:id/import/:importId', passport.csrfProtection, (req,
}
subscriptions.getImport(req.params.id, req.params.importId, (err, data) => {
if (err || !list) {
if (err || !data) {
req.flash('danger', err && err.message || err || 'Could not find import data with specified ID');
return res.redirect('/lists');
}
@ -597,7 +598,9 @@ router.post('/subscription/import-restart', passport.parseForm, passport.csrfPro
status: 1,
error: null,
finished: null,
processed: 0
processed: 0,
new: 0,
failed: 0
}, (err, importer) => {
if (err || !importer) {
req.flash('danger', err && err.message || err || 'Could not find import data with specified ID');
@ -610,4 +613,35 @@ router.post('/subscription/import-restart', passport.parseForm, passport.csrfPro
});
});
router.get('/subscription/:id/import/:importId/failed', (req, res) => {
let start = 0;
lists.get(req.params.id, (err, list) => {
if (err || !list) {
req.flash('danger', err && err.message || err || 'Could not find list with specified ID');
return res.redirect('/lists');
}
subscriptions.getImport(req.params.id, req.params.importId, (err, data) => {
if (err || !data) {
req.flash('danger', err && err.message || err || 'Could not find import data with specified ID');
return res.redirect('/lists');
}
subscriptions.getFailedImports(req.params.importId, (err, rows) => {
if (err) {
req.flash('danger', err && err.message || err);
return res.redirect('/lists');
}
data.rows = rows.map((row, i) => {
row.index = start + i + 1;
return row;
});
data.list = list;
res.render('lists/subscription/import-failed', data);
});
});
});
});
module.exports = router;

38
services/importer.js
View file

@ -139,18 +139,38 @@ function processImport(data, callback) {
tools.validateEmail(entry.email, true, err => {
if (err) {
log.verbose('Import', 'Failed processing row %s: %s', entry.email, err.message);
return setImmediate(processRows);
let reason = (err.message || '').toString().trim().replace(/^[a-z]Error:\s*/i, '');
log.verbose('Import', 'Failed processing row %s: %s', entry.email, reason);
db.getConnection((err, connection) => {
if (err) {
log.error('Import', err.stack);
return setImmediate(processRows);
}
let query = 'INSERT INTO import_failed (`import`, `email`, `reason`) VALUES(?,?,?)';
connection.query(query, [data.id, entry.email, reason], err => {
if (err) {
connection.release();
return setImmediate(processRows);
}
let query = 'UPDATE importer SET `failed`=`failed`+1 WHERE `id`=? LIMIT 1';
connection.query(query, [data.id], () => {
connection.release();
return setImmediate(processRows);
});
});
});
return;
}
subscriptions.insert(listId, {
imported: data.id,
status: data.type
}, entry, (err, entryId) => {
}, entry, (err, response) => {
if (err) {
// ignore
log.error('Import', err.stack);
} else if (entryId) {
} else if (response.entryId) {
//log.verbose('Import', 'Inserted %s as %s', entry.email, entryId);
}
@ -160,7 +180,15 @@ function processImport(data, callback) {
return setImmediate(processRows);
}
let query = 'UPDATE importer SET `processed`=`processed`+1 WHERE `id`=? LIMIT 1';
let query;
if (response.inserted) {
// this record did not exist before, count as new
query = 'UPDATE importer SET `processed`=`processed`+1, `new`=`new`+1 WHERE `id`=? LIMIT 1';
} else {
// it's an existing record
query = 'UPDATE importer SET `processed`=`processed`+1 WHERE `id`=? LIMIT 1';
}
connection.query(query, [data.id], () => {
connection.release();
return setImmediate(processRows);

4
setup/sql/mailtrain.sql
View file

@ -157,7 +157,7 @@ CREATE TABLE `segments` (
`created` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `list` (`list`),
KEY `name` (`name`(191)),
KEY `name` (`name`),
CONSTRAINT `segments_ibfk_1` FOREIGN KEY (`list`) REFERENCES `lists` (`id`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
@ -222,7 +222,7 @@ CREATE TABLE `users` (
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
LOCK TABLES `users` WRITE;
INSERT INTO `users` VALUES (1,'admin','$2a$10$mzKU71G62evnGB2PvQA4k..Wf9jASk.c7a8zRMHh6qQVjYJ2r/g/K','admin@example.com',NULL,NULL,'2016-04-20 17:20:48');
INSERT INTO `users` VALUES (1,'admin','$2a$10$mzKU71G62evnGB2PvQA4k..Wf9jASk.c7a8zRMHh6qQVjYJ2r/g/K','admin@example.com',NULL,NULL,NOW());
UNLOCK TABLES;
SET UNIQUE_CHECKS=1;

12
setup/sql/upgrade-00002.sql Normal file
View file

@ -0,0 +1,12 @@
# Header section
# Define incrementing schema version number
SET @schema_version = '2';
# Adds new column 'failed' to importer table. Includes the count of failed addresses for an import
ALTER TABLE importer ADD COLUMN `failed` INT(11) UNSIGNED NOT NULL DEFAULT '0' AFTER `processed`;
ALTER TABLE importer ADD COLUMN `new` INT(11) UNSIGNED NOT NULL DEFAULT '0' AFTER `processed`;
# Footer section
LOCK TABLES `settings` WRITE;
INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
UNLOCK TABLES;

44
views/lists/subscription/import-failed.hbs Normal file
View file

@ -0,0 +1,44 @@
<ol class="breadcrumb">
<li><a href="/">Home</a></li>
<li><a href="/lists/">Lists</a></li>
<li><a href="/lists/view/{{list.id}}">{{list.name}}</a></li>
<li class="active">Import status</li>
</ol>
<h2>{{list.name}} <small>Failed addresses</small> <a class="btn btn-default btn-xs" href="/lists/view/{{list.id}}" role="button"><span class="glyphicon glyphicon-arrow-left" aria-hidden="true"></span> Back to list</a></h2>
<hr>
<div class="table-responsive">
<table class="table table-bordered table-hover display nowrap" width="100%">
<thead>
<th class="col-md-1">
#
</th>
<th>
Address
</th>
<th>
Fail reason
</th>
</thead>
{{#if rows}}
<tbody>
{{#each rows}}
<tr>
<th scope="row">
{{index}}
</th>
<td>
<a href="mailto:{{email}}">{{email}}</a>
</a>
</td>
<td>
<span class="text-danger">{{reason}}</span>
</td>
</tr>
{{/each}}
</tbody>
{{/if}}
</table>
</div>

18
views/lists/view.hbs
View file

@ -121,7 +121,13 @@
Type
</th>
<th>
Processed
Added
</th>
<th>
Updated
</th>
<th>
Failed
</th>
<th>
Status
@ -154,7 +160,13 @@
{{importType}}
</td>
<td>
{{processed}}
{{new}}
</td>
<td>
{{updated}}
</td>
<td>
{{#if failed}}<a href="/lists/subscription/{{../id}}/import/{{id}}/failed">{{failed}}</a>{{else}} 0 {{/if}}
</td>
<td class="{{#if error}}text-danger{{/if}}">
{{#if error}}
@ -175,7 +187,7 @@
{{/each}}
{{else}}
<tr>
<td colspan="7">
<td colspan="9">
No data available in table
</td>
</tr>