From 0b4e8666635481db596ad0d42165c9fa8e2b481a Mon Sep 17 00:00:00 2001 From: Luc LosCan Date: Tue, 27 Jun 2017 18:22:12 +0200 Subject: [PATCH 1/9] Check for locally requeued messages in postfix --- services/postfix-bounce-server.js | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/services/postfix-bounce-server.js b/services/postfix-bounce-server.js index 60aba027..d29b60e9 100644 --- a/services/postfix-bounce-server.js +++ b/services/postfix-bounce-server.js @@ -6,6 +6,7 @@ let net = require('net'); let campaigns = require('../lib/models/campaigns'); let seenIds = new Set(); +let queueIds = {}; let server = net.createServer(socket => { let remainder = ''; @@ -28,7 +29,7 @@ let server = net.createServer(socket => { return readNextChunk(); } let line = lines[pos++]; - let match = /\bstatus=bounced\b/.test(line) && line.match(/\bpostfix\/\w+\[\d+\]:\s*([^:]+)/); + let match = /\bstatus=(bounced|sent)\b/.test(line) && line.match(/\bpostfix\/\w+\[\d+\]:\s*([^:]+).*?status=(\w+)/); if (match) { let queueId = match[1]; @@ -37,6 +38,23 @@ let server = net.createServer(socket => { } seenIds.add(queueId); + // Losacno: Check for local requeue + let status = match[2]; + log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status); + if ( status == 'sent' ) { + let queued = / relay=127\.0\.0\.1/.test(line) && line.match(/ queued as (\w+)\)/); + if ( queued ) { + log.verbose('POSTFIXBOUNCE', 'Marked message %s as locally requeued as %s', queueId, queued[1]); + queueIds[queued[1]] = queueId; + } + return checkNextLine(); + } else { + if ( queueId in queueIds ) { + log.verbose('POSTFIXBOUNCE', 'Message %s was requeued from %s', queueId, queueIds[queueId]); + queueId = queueIds[queueId]; + } + } + campaigns.findMailByResponse(queueId, (err, message) => { if (err || !message) { return checkNextLine(); From 219c77606d801ebee77fa038267112d5747cf796 Mon Sep 17 00:00:00 2001 From: Luc LosCan Date: Wed, 28 Jun 2017 10:31:08 +0200 Subject: [PATCH 2/9] Fix Travic CI failing checks --- services/postfix-bounce-server.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/services/postfix-bounce-server.js b/services/postfix-bounce-server.js index d29b60e9..f41602cf 100644 --- a/services/postfix-bounce-server.js +++ b/services/postfix-bounce-server.js @@ -41,18 +41,16 @@ let server = net.createServer(socket => { // Losacno: Check for local requeue let status = match[2]; log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status); - if ( status == 'sent' ) { + if ( status === 'sent' ) { let queued = / relay=127\.0\.0\.1/.test(line) && line.match(/ queued as (\w+)\)/); if ( queued ) { log.verbose('POSTFIXBOUNCE', 'Marked message %s as locally requeued as %s', queueId, queued[1]); queueIds[queued[1]] = queueId; } return checkNextLine(); - } else { - if ( queueId in queueIds ) { - log.verbose('POSTFIXBOUNCE', 'Message %s was requeued from %s', queueId, queueIds[queueId]); - queueId = queueIds[queueId]; - } + } else if ( queueId in queueIds ) { + log.verbose('POSTFIXBOUNCE', 'Message %s was requeued from %s', queueId, queueIds[queueId]); + queueId = queueIds[queueId]; } campaigns.findMailByResponse(queueId, (err, message) => { From f90e67d775900633fb006efce8c3e7c9d92a125f Mon Sep 17 00:00:00 2001 From: Luc LosCan Date: Mon, 10 Jul 2017 12:13:17 +0200 Subject: [PATCH 3/9] Save new queueId/response on locally requeued messages --- lib/models/campaigns.js | 18 ++++++++++++ services/postfix-bounce-server.js | 47 ++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index 0fb4323d..bfef231f 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -1086,6 +1086,24 @@ module.exports.updateMessage = (message, status, updateSubscription, callback) = }); }; +module.exports.updateMessageResponse = (message, response, response_id, callback) => { + db.getConnection((err, connection) => { + if (err) { + return callback(err); + } + + let query = 'UPDATE `campaign__' + message.campaign + '` SET `response`=?, `response_id`=? WHERE id=? LIMIT 1'; + connection.query(query, [response, response_id, message.id], err => { + connection.release(); + if (err) { + return callback(err); + } + return callback(null, true); + }); + + }); +}; + function createCampaignTables(id, callback) { let query = 'CREATE TABLE `campaign__' + id + '` LIKE campaign'; db.getConnection((err, connection) => { diff --git a/services/postfix-bounce-server.js b/services/postfix-bounce-server.js index f41602cf..d08233c7 100644 --- a/services/postfix-bounce-server.js +++ b/services/postfix-bounce-server.js @@ -6,7 +6,6 @@ let net = require('net'); let campaigns = require('../lib/models/campaigns'); let seenIds = new Set(); -let queueIds = {}; let server = net.createServer(socket => { let remainder = ''; @@ -32,6 +31,8 @@ let server = net.createServer(socket => { let match = /\bstatus=(bounced|sent)\b/.test(line) && line.match(/\bpostfix\/\w+\[\d+\]:\s*([^:]+).*?status=(\w+)/); if (match) { let queueId = match[1]; + let queued = ''; + let queued_as = ''; if (seenIds.has(queueId)) { return checkNextLine(); @@ -42,28 +43,42 @@ let server = net.createServer(socket => { let status = match[2]; log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status); if ( status === 'sent' ) { - let queued = / relay=127\.0\.0\.1/.test(line) && line.match(/ queued as (\w+)\)/); - if ( queued ) { - log.verbose('POSTFIXBOUNCE', 'Marked message %s as locally requeued as %s', queueId, queued[1]); - queueIds[queued[1]] = queueId; + // Save new queueId to update message's previous queueId (thanks @mfechner ) + if ( queued = / relay=127\.0\.0\.1/.test(line) && line.match(/status=sent \((.*)\)/) ) { + queued = queued[1]; + queued_as = queued.match(/ queued as (\w+)/); + queued_as = queued_as[1]; } - return checkNextLine(); - } else if ( queueId in queueIds ) { - log.verbose('POSTFIXBOUNCE', 'Message %s was requeued from %s', queueId, queueIds[queueId]); - queueId = queueIds[queueId]; } campaigns.findMailByResponse(queueId, (err, message) => { if (err || !message) { return checkNextLine(); } - campaigns.updateMessage(message, 'bounced', true, (err, updated) => { - if (err) { - log.error('POSTFIXBOUNCE', 'Failed updating message: %s', err && err.stack); - } else if (updated) { - log.verbose('POSTFIXBOUNCE', 'Marked message %s as bounced', queueId); - } - }); + if ( queued_as ) { + log.verbose('POSTFIXBOUNCE', 'Message %s locally requeued as %s', queueId, queued_as); + // Update message's previous queueId (thanks @mfechner ) + campaigns.updateMessageResponse(message, queued, queued_as, (err, updated) => { + if (err) { + log.error('POSTFIXBOUNCE', 'Failed updating message: %s', err && err.stack); + } else if (updated) { + log.verbose('POSTFIXBOUNCE', 'Successfully changed message queueId to %s', queued_as); + } + }); + + } else { + campaigns.updateMessage(message, 'bounced', true, (err, updated) => { + if (err) { + log.error('POSTFIXBOUNCE', 'Failed updating message: %s', err && err.stack); + } else if (updated) { + log.verbose('POSTFIXBOUNCE', 'Marked message %s as bounced', queueId); + } + }); + } + + // No need to keep in memory... free it ( thanks @witzig ) + seenIds.delete(queueId); + return checkNextLine(); }); return; From 3e04f82fe257c6a2a937ead7d4871e5736603af5 Mon Sep 17 00:00:00 2001 From: Luc LosCan Date: Mon, 10 Jul 2017 12:22:01 +0200 Subject: [PATCH 4/9] Fix Travis CI checks --- services/postfix-bounce-server.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/postfix-bounce-server.js b/services/postfix-bounce-server.js index d08233c7..6585cb7f 100644 --- a/services/postfix-bounce-server.js +++ b/services/postfix-bounce-server.js @@ -44,7 +44,8 @@ let server = net.createServer(socket => { log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status); if ( status === 'sent' ) { // Save new queueId to update message's previous queueId (thanks @mfechner ) - if ( queued = / relay=127\.0\.0\.1/.test(line) && line.match(/status=sent \((.*)\)/) ) { + queued = / relay=127\.0\.0\.1/.test(line) && line.match(/status=sent \((.*)\)/); + if ( queued ) { queued = queued[1]; queued_as = queued.match(/ queued as (\w+)/); queued_as = queued_as[1]; From 3ab718705e1a76c4f9f9d0464de4305ad56df56f Mon Sep 17 00:00:00 2001 From: Luc LosCan Date: Mon, 10 Jul 2017 17:14:36 +0200 Subject: [PATCH 5/9] Removed 127.0.0.1 from queued .test() --- services/postfix-bounce-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/postfix-bounce-server.js b/services/postfix-bounce-server.js index 6585cb7f..3bc578b9 100644 --- a/services/postfix-bounce-server.js +++ b/services/postfix-bounce-server.js @@ -44,7 +44,7 @@ let server = net.createServer(socket => { log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status); if ( status === 'sent' ) { // Save new queueId to update message's previous queueId (thanks @mfechner ) - queued = / relay=127\.0\.0\.1/.test(line) && line.match(/status=sent \((.*)\)/); + queued = / relay=/.test(line) && line.match(/status=sent \((.*)\)/); if ( queued ) { queued = queued[1]; queued_as = queued.match(/ queued as (\w+)/); From ce2ed26fc991cdb86aba2db31f9752dd5b3a5ea0 Mon Sep 17 00:00:00 2001 From: witzig Date: Thu, 13 Jul 2017 22:29:55 +0200 Subject: [PATCH 6/9] findMailByResponse now searches in latest campaigns first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should be more efficient especially if there’re many campaigns. --- lib/models/campaigns.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index bfef231f..8d7bc899 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -951,7 +951,7 @@ module.exports.findMailByResponse = (responseId, callback) => { return callback(err); } - connection.query('SELECT id FROM campaigns', [], (err, campaignList) => { + connection.query('SELECT id FROM campaigns ORDER BY id DESC', [], (err, campaignList) => { if (err) { connection.release(); return callback(err); From b57615543b49ee64c75e6101172ae02d09ba1571 Mon Sep 17 00:00:00 2001 From: witzig Date: Thu, 13 Jul 2017 22:45:35 +0200 Subject: [PATCH 7/9] Fixed Travis CI Checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Travis CI didn’t install the latest 7.x release somehow. Testing with 8.x henceforth. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0a827b44..dc5ea4b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ dist: trusty sudo: required language: node_js node_js: - - 7 + - 8 services: - mysql before_install: From 4cede37e9d337e8fbdaaeb3f82d2ec2f51607f56 Mon Sep 17 00:00:00 2001 From: witzig Date: Fri, 14 Jul 2017 23:48:18 +0200 Subject: [PATCH 8/9] Fixes #278 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the bounced counter is correctly increased, the status of failed messages has been set to 'unsubscribed' instead of 'bounced‘. --- services/sender.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/sender.js b/services/sender.js index 040a8a13..a30b38ee 100644 --- a/services/sender.js +++ b/services/sender.js @@ -523,7 +523,7 @@ let sendLoop = () => { } } - let status = err ? 2 : 1; + let status = err ? 3 : 1; let response = err && (err.response || err.message) || info.response || info.messageId; let responseId = response.split(/\s+/).pop(); @@ -533,7 +533,7 @@ let sendLoop = () => { return; } - let query = 'UPDATE `campaigns` SET `delivered`=`delivered`+1 ' + (status === 2 ? ', `bounced`=`bounced`+1 ' : '') + ' WHERE id=? LIMIT 1'; + let query = 'UPDATE `campaigns` SET `delivered`=`delivered`+1 ' + (status === 3 ? ', `bounced`=`bounced`+1 ' : '') + ' WHERE id=? LIMIT 1'; connection.query(query, [message.campaignId], err => { if (err) { From 98ab33ccfeb3400aa3d927d6b517dcbc0841fb7b Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 21 Jul 2017 15:47:10 +0200 Subject: [PATCH 9/9] Fix docker install instructions Change extension name of copied docker- compose-override file to not be the same as source --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 60726293..a25979ba 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ With proper SPF, DKIM and PTR records (DMARC wouldn't hurt either) I got perfect * Download Mailtrain files using git: `git clone git://github.com/Mailtrain-org/mailtrain.git` (or download [zipped repo](https://github.com/Mailtrain-org/mailtrain/archive/master.zip)) and open Mailtrain folder `cd mailtrain` * **Note**: depending on how you have configured your system and Docker you may need to prepend the commands below with `sudo`. -* Copy the file `docker-compose.override.yml.tmpl` to `docker-compose.override.yml.tmpl` and modify it if you need to. +* Copy the file `docker-compose.override.yml.tmpl` to `docker-compose.override.yml` and modify it if you need to. * Bring up the stack with: `docker-compose up -d`, by default it will use the included `docker-compose.yml` file and override some configurations taken from the `docker-compose.override.yml` file. * If you want to use only / copy the `docker-compose.yml` file (for example, if you were deploying with Rancher), you may need to first run `docker-compose build` to make sure your system has a Docker image `mailtrain:latest`. * Open [http://localhost:3000/](http://localhost:3000/) (change the host name `localhost` to the name of the host where you are deploying the system).