Fixed throttling and pausing #243

lib/db.js

@@ -4,6 +4,7 @@ let config = require('config');
 let mysql = require('mysql');
 let redis = require('redis');
 let Lock = require('redfour');
+let tools = require('./tools');
 
 module.exports = mysql.createPool(config.mysql);
 if (config.redis && config.redis.enabled) {
@@ -33,7 +34,7 @@ if (config.redis && config.redis.enabled) {
     };
 
     module.exports.clearCache = (key, callback) => {
-        module.exports.redis.del(key, err => callback(err));
+        module.exports.redis.del('mailtrain:cache:' + key, err => callback(err));
     };
 
     module.exports.addToCache = (key, value, callback) => {
@@ -76,9 +77,21 @@ if (config.redis && config.redis.enabled) {
 
     module.exports.clearCache = (key, callback) => {
         caches.delete(key);
+        tools.workers.forEach(child => {
+            child.send({
+                cmd: 'db.clearCache',
+                key
+            });
+        });
         setImmediate(() => callback());
     };
 
+    process.on('message', m => {
+        if (m && m.cmd === 'db.clearCache' && m.key) {
+            caches.delete(m.key);
+        }
+    });
+
     module.exports.addToCache = (key, value, callback) => {
         if (!caches.has(key)) {
             caches.set(key, []);

lib/mailer.js

@@ -182,10 +182,10 @@ function createMailer(callback) {
             module.exports.transport.checkThrottling = null;
         }
 
-        let throttling = Number(configItems.smtpThrottling) || 0;
+        let sendingRate = Number(configItems.smtpThrottling) || 0;
-        if (throttling) {
+        if (sendingRate) {
             // convert to messages/second
-            throttling = 1 / (throttling / (3600 * 1000));
+            sendingRate = sendingRate / 3600;
         }
 
         let transportOptions;
@@ -236,7 +236,7 @@ function createMailer(callback) {
                     error: logfunc.bind(null, 'error')
                 },
                 maxConnections: Number(configItems.smtpMaxConnections),
-                sendingRate: throttling,
+                sendingRate,
                 tls: {
                     rejectUnauthorized: !configItems.smtpSelfSigned
                 }
@@ -257,19 +257,30 @@ function createMailer(callback) {
             oldListeners.forEach(listener => module.exports.transport.on('idle', listener));
         }
 
-        let lastCheck = Date.now();
+
         if (configItems.mailTransport === 'smtp' || !configItems.mailTransport) {
+
+            let throttling = Number(configItems.smtpThrottling) || 0;
+            if (throttling) {
+                throttling = 1 / (throttling / (3600 * 1000));
+            }
+
+            let lastCheck = Date.now();
+
             module.exports.transport.checkThrottling = function (next) {
                 if (!throttling) {
                     return next();
                 }
                 let nextCheck = Date.now();
                 let checkDiff = (nextCheck - lastCheck);
-                lastCheck = nextCheck;
                 if (checkDiff < throttling) {
                     log.verbose('Mail', 'Throttling next message in %s sec.', (throttling - checkDiff) / 1000);
-                    setTimeout(next, throttling - checkDiff);
+                    setTimeout(() => {
+                        lastCheck = Date.now();
+                        next();
+                    }, throttling - checkDiff);
                 } else {
+                    lastCheck = nextCheck;
                     next();
                 }
             };

services/sender.js

@@ -466,12 +466,20 @@ let sendLoop = () => {
             return setTimeout(sendLoop, 10 * 1000);
         }
 
+        let isThrottled = false;
+
         let getNext = () => {
-            if (!mailer.transport.isIdle()) {
+            if (!mailer.transport.isIdle() || isThrottled) {
                 // only retrieve new messages if there are free slots in the mailer queue
                 return;
             }
 
+            isThrottled = true;
+
+            mailer.transport.checkThrottling(() => {
+
+                isThrottled = false;
+
                 // find an unsent message
                 findUnsent((err, message) => {
                     if (err) {
@@ -484,7 +492,7 @@ let sendLoop = () => {
                         return;
                     }
 
-                //log.verbose('Mail', 'Found new message to be delivered: %s', message.subscription.cid);
+                    // log.verbose('Mail', 'Found new message to be delivered: %s', message.subscription.cid);
                     // format message to nodemailer message format
                     formatMessage(message, (err, mail) => {
                         if (err) {
@@ -499,6 +507,7 @@ let sendLoop = () => {
                                 setTimeout(getNext, mailing_timeout);
                                 return;
                             }
+
                             if (!blacklisted) {
                                 let tryCount = 0;
                                 let trySend = () => {
@@ -571,14 +580,15 @@ let sendLoop = () => {
                                     });
                                 });
                             }
-                      setImmediate(() => mailer.transport.checkThrottling(getNext));
+                            setImmediate(getNext);
+                        });
                     });
                 });
             });
         };
 
-        mailer.transport.on('idle', () => mailer.transport.checkThrottling(getNext));
+        mailer.transport.on('idle', getNext);
-        setImmediate(() => mailer.transport.checkThrottling(getNext));
+        setImmediate(getNext);
     });
 };