Preparation of merge with master

lib/executor.js

@@ -27,7 +27,7 @@ function spawn(callback) {
             if (msg.type === 'process-started') {
                 let requestCallback = requestCallbacks[msg.tid];
                 if (requestCallback && requestCallback.startedCallback) {
-                    requestCallback.startedCallback(msg.tid);
+                    requestCallback.startedCallback(msg.tid, );
                 }
 
             } else if (msg.type === 'process-failed') {

lib/importer.js

@@ -0,0 +1,54 @@
+'use strict';
+
+const knex = require('./knex');
+const fork = require('child_process').fork;
+const log = require('npmlog');
+const path = require('path');
+const {ImportStatus} = require('../shared/imports');
+
+let messageTid = 0;
+let importerProcess;
+
+module.exports = {
+    spawn,
+    scheduleCheck
+};
+
+function spawn(callback) {
+    log.info('Importer', 'Spawning importer process.');
+
+    knex.transaction(async tx => {
+        await tx('imports').where('status', ImportStatus.PREP_RUNNING).update({status: ImportStatus.PREP_SCHEDULED});
+        await tx('imports').where('status', ImportStatus.RUN_RUNNING).update({status: ImportStatus.RUN_SCHEDULED});
+
+    }).then(() => {
+        importerProcess = fork(path.join(__dirname, '..', 'services', 'importer.js'), [], {
+            cwd: path.join(__dirname, '..'),
+            env: {NODE_ENV: process.env.NODE_ENV}
+        });
+
+        importerProcess.on('message', msg => {
+            if (msg) {
+                if (msg.type === 'importer-started') {
+                    log.info('Importer', 'Importer process started.');
+                    return callback();
+                }
+            }
+        });
+
+        importerProcess.on('close', (code, signal) => {
+            log.info('Importer', 'Importer process exited with code %s signal %s.', code, signal);
+        });
+    });
+}
+
+function scheduleCheck() {
+    importerProcess.send({
+        type: 'scheduleCheck',
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+

lib/senders.js

@@ -1,50 +0,0 @@
-'use strict';
-
-const fork = require('child_process').fork;
-
-const config = require('config');
-const log = require('npmlog');
-const workers = new Set();
-
-function spawn(callback) {
-    let processes = Math.max(Number(config.queue.processes) || 1, 1);
-    let spawned = 0;
-    let returned = false;
-
-    if (processes > 1 && !config.redis.enabled) {
-        log.error('Queue', '%s processes requested but Redis is not enabled, spawning 1 process', processes);
-        processes = 1;
-    }
-
-    let spawnSender = function () {
-        if (spawned >= processes) {
-            if (!returned) {
-                returned = true;
-                return callback();
-            }
-            return false;
-        }
-
-        let child = fork(__dirname + '/../services/sender.js', []);
-        let pid = child.pid;
-        workers.add(child);
-
-        child.on('close', (code, signal) => {
-            spawned--;
-            workers.delete(child);
-            log.error('Child', 'Sender process %s exited with %s', pid, code || signal);
-            // Respawn after 5 seconds
-            setTimeout(() => spawnSender(), 5 * 1000).unref();
-        });
-
-        spawned++;
-        setImmediate(spawnSender);
-    };
-
-    spawnSender();
-}
-
-module.exports = {
-    workers,
-    spawn
-};