Work in progress on securing reports.
This commit is contained in:
Tomas Bures
parent
3072632d8d
commit
418dba7b9f
14 changed files with 709 additions and 331 deletions
74
lib/executor.js
Normal file
74
lib/executor.js
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
'use strict';
|
||||
|
||||
const fork = require('child_process').fork;
|
||||
const log = require('npmlog');
|
||||
const path = require('path');
|
||||
|
||||
const requestCallbacks = {};
|
||||
let messageTid = 0;
|
||||
let executorProcess;
|
||||
|
||||
module.exports = {
|
||||
spawn,
|
||||
start,
|
||||
stop
|
||||
};
|
||||
|
||||
function spawn(callback) {
|
||||
log.info('Executor', 'Spawning executor process.');
|
||||
|
||||
executorProcess = fork(path.join(__dirname, '..', 'services', 'executor.js'), [], {
|
||||
cwd: path.join(__dirname, '..'),
|
||||
env: {NODE_ENV: process.env.NODE_ENV}
|
||||
});
|
||||
|
||||
executorProcess.on('message', msg => {
|
||||
if (msg) {
|
||||
if (msg.type === 'process-started') {
|
||||
let requestCallback = requestCallbacks[msg.tid];
|
||||
if (requestCallback && requestCallback.startedCallback) {
|
||||
requestCallback.startedCallback(msg.tid);
|
||||
}
|
||||
|
||||
} else if (msg.type === 'process-finished') {
|
||||
let requestCallback = requestCallbacks[msg.tid];
|
||||
if (requestCallback && requestCallback.startedCallback) {
|
||||
requestCallback.finishedCallback(msg.code, msg.signal);
|
||||
}
|
||||
|
||||
delete requestCallbacks[msg.tid];
|
||||
|
||||
} else if (msg.type === 'executor-started') {
|
||||
log.info('Executor', 'Executor process started.');
|
||||
return callback();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
executorProcess.on('close', (code, signal) => {
|
||||
log.info('Executor', 'Executor process exited with code %s signal %s.', code, signal);
|
||||
});
|
||||
}
|
||||
|
||||
function start(type, data, startedCallback, finishedCallback) {
|
||||
requestCallbacks[messageTid] = {
|
||||
startedCallback,
|
||||
finishedCallback
|
||||
};
|
||||
|
||||
executorProcess.send({
|
||||
type: 'start-' + type,
|
||||
data,
|
||||
tid: messageTid
|
||||
});
|
||||
|
||||
messageTid++;
|
||||
}
|
||||
|
||||
function stop(tid) {
|
||||
executorProcess.send({
|
||||
type: 'stop-process',
|
||||
tid
|
||||
});
|
||||
}
|
||||
|
||||
33
lib/file-helpers.js
Normal file
33
lib/file-helpers.js
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
|
||||
function nameToFileName(name) {
|
||||
return name.
|
||||
trim().
|
||||
toLowerCase().
|
||||
replace(/[ .+/]/g, '-').
|
||||
replace(/[^a-z0-9\-_]/gi, '').
|
||||
replace(/--*/g, '-');
|
||||
}
|
||||
|
||||
|
||||
function getReportDir(report) {
|
||||
return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + nameToFileName(report.name));
|
||||
}
|
||||
|
||||
function getReportContentFile(report) {
|
||||
return path.join(getReportDir(report), 'report');
|
||||
}
|
||||
|
||||
function getReportOutputFile(report) {
|
||||
return getReportDir(report) + '.output';
|
||||
}
|
||||
|
||||
|
||||
module.exports = {
|
||||
getReportContentFile,
|
||||
getReportDir,
|
||||
getReportOutputFile,
|
||||
nameToFileName
|
||||
};
|
||||
131
lib/privilege-helpers.js
Normal file
131
lib/privilege-helpers.js
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
'use strict';
|
||||
|
||||
const log = require('npmlog');
|
||||
const config = require('config');
|
||||
const path = require('path');
|
||||
|
||||
const promise = require('bluebird');
|
||||
const fsExtra = promise.promisifyAll(require('fs-extra'));
|
||||
const fs = promise.promisifyAll(require('fs'));
|
||||
const walk = require('walk');
|
||||
|
||||
const tryRequire = require('try-require');
|
||||
const posix = tryRequire('posix');
|
||||
|
||||
|
||||
function ensureMailtrainOwner(file, callback) {
|
||||
try {
|
||||
const uid = config.user ? posix.getpwnam(config.user).uid : 0;
|
||||
const gid = config.group ? posix.getgrnam(config.group).gid : 0;
|
||||
|
||||
fs.chown(file, uid, gid, callback);
|
||||
|
||||
} catch (err) {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureMailtrainOwnerRecursive(dir, callback) {
|
||||
try {
|
||||
const uid = config.user ? posix.getpwnam(config.user).uid : 0;
|
||||
const gid = config.group ? posix.getgrnam(config.group).gid : 0;
|
||||
|
||||
fs.chown(dir, uid, gid, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
walk.walk(dir)
|
||||
.on('node', (root, stat, next) => {
|
||||
fs.chown(path.join(root, stat.name), uid, gid, next);
|
||||
})
|
||||
.on('end', callback);
|
||||
});
|
||||
} catch (err) {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
|
||||
const ensureMailtrainOwnerRecursiveAsync = promise.promisify(ensureMailtrainOwnerRecursive);
|
||||
|
||||
function dropRootPrivileges() {
|
||||
if (config.group) {
|
||||
try {
|
||||
process.setgid(config.group);
|
||||
log.info('PrivilegeHelpers', 'Changed group to "%s" (%s)', config.group, process.getgid());
|
||||
} catch (E) {
|
||||
log.info('PrivilegeHelpers', 'Failed to change group to "%s" (%s)', config.group, E.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (config.user) {
|
||||
try {
|
||||
process.setuid(config.user);
|
||||
log.info('PrivilegeHelpers', 'Changed user to "%s" (%s)', config.user, process.getuid());
|
||||
} catch (E) {
|
||||
log.info('PrivilegeHelpers', 'Failed to change user to "%s" (%s)', config.user, E.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function setupChrootDir(newRoot, callback) {
|
||||
try {
|
||||
fsExtra.emptyDirAsync(newRoot)
|
||||
.then(() => fsExtra.ensureDirAsync(path.join(newRoot, 'etc')))
|
||||
.then(() => fsExtra.copyAsync('/etc/hosts', path.join(newRoot, 'etc', 'hosts')))
|
||||
.then(() => ensureMailtrainOwnerRecursiveAsync(newRoot))
|
||||
.then(() => {
|
||||
log.info('PrivilegeHelpers', 'Chroot directory "%s" set up', newRoot);
|
||||
callback();
|
||||
})
|
||||
.catch(err => {
|
||||
log.info('PrivilegeHelpers', 'Failed to setup chroot directory "%s"', newRoot);
|
||||
callback(err);
|
||||
});
|
||||
|
||||
} catch(err) {
|
||||
log.info('PrivilegeHelpers', 'Failed to setup chroot directory "%s"', newRoot);
|
||||
}
|
||||
}
|
||||
|
||||
function tearDownChrootDir(root, callback) {
|
||||
if (posix) {
|
||||
fsExtra.removeAsync(path.join('/', 'etc'))
|
||||
.then(() => {
|
||||
log.info('PrivilegeHelpers', 'Chroot directory "%s" torn down', root);
|
||||
callback();
|
||||
})
|
||||
.catch(err => {
|
||||
log.info('PrivilegeHelpers', 'Failed to tear down chroot directory "%s"', root);
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function chrootAndDropRootPrivileges(newRoot) {
|
||||
|
||||
try {
|
||||
const uid = config.user ? posix.getpwnam(config.user).uid : 0;
|
||||
const gid = config.group ? posix.getgrnam(config.group).gid : 0;
|
||||
|
||||
posix.chroot(newRoot);
|
||||
process.chdir('/');
|
||||
|
||||
process.setgid(gid);
|
||||
process.setuid(uid);
|
||||
|
||||
log.info('PrivilegeHelpers', 'Changed root to "%s" and privileges to %s.%s', newRoot, uid, gid);
|
||||
} catch(err) {
|
||||
log.info('PrivilegeHelpers', 'Failed to change root to "%s" and set privileges', newRoot);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dropRootPrivileges,
|
||||
chrootAndDropRootPrivileges,
|
||||
setupChrootDir,
|
||||
tearDownChrootDir,
|
||||
ensureMailtrainOwner,
|
||||
ensureMailtrainOwnerRecursive
|
||||
};
|
||||
129
lib/report-processor.js
Normal file
129
lib/report-processor.js
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
'use strict';
|
||||
|
||||
const log = require('npmlog');
|
||||
const reports = require('./models/reports');
|
||||
const executor = require('./executor');
|
||||
|
||||
let runningWorkersCount = 0;
|
||||
let maxWorkersCount = 1;
|
||||
|
||||
let workers = {};
|
||||
|
||||
function startWorker(report) {
|
||||
|
||||
function onStarted(tid) {
|
||||
log.info('ReportProcessor', 'Worker process for "%s" started with tid %s. Current worker count is %s.', report.name, tid, runningWorkersCount);
|
||||
workers[report.id] = tid;
|
||||
}
|
||||
|
||||
function onFinished(code, signal) {
|
||||
runningWorkersCount--;
|
||||
log.info('ReportProcessor', 'Worker process for "%s" (tid %s) exited with code %s signal %s. Current worker count is %s.', report.name, workers[report.id], code, signal, runningWorkersCount);
|
||||
delete workers[report.id];
|
||||
|
||||
const fields = {};
|
||||
if (code === 0) {
|
||||
fields.state = reports.ReportState.FINISHED;
|
||||
fields.lastRun = new Date();
|
||||
} else {
|
||||
fields.state = reports.ReportState.FAILED;
|
||||
}
|
||||
|
||||
reports.updateFields(report.id, fields, err => {
|
||||
if (err) {
|
||||
log.error('ReportProcessor', err);
|
||||
}
|
||||
|
||||
setImmediate(startWorkers);
|
||||
});
|
||||
}
|
||||
|
||||
const reportData = {
|
||||
id: report.id,
|
||||
name: report.name
|
||||
};
|
||||
|
||||
runningWorkersCount++;
|
||||
executor.start('report-processor-worker', reportData, onStarted, onFinished);
|
||||
}
|
||||
|
||||
function startWorkers() {
|
||||
reports.listWithState(reports.ReportState.SCHEDULED, 0, maxWorkersCount - runningWorkersCount, (err, reportList) => {
|
||||
if (err) {
|
||||
log.error('ReportProcessor', err);
|
||||
return;
|
||||
}
|
||||
|
||||
for (let report of reportList) {
|
||||
reports.updateFields(report.id, { state: reports.ReportState.PROCESSING }, err => {
|
||||
if (err) {
|
||||
log.error('ReportProcessor', err);
|
||||
return;
|
||||
}
|
||||
|
||||
startWorker(report);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports.start = (reportId, callback) => {
|
||||
if (!workers[reportId]) {
|
||||
log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
|
||||
reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (runningWorkersCount < maxWorkersCount) {
|
||||
log.info('ReportProcessor', 'Starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
|
||||
|
||||
startWorkers();
|
||||
} else {
|
||||
log.info('ReportProcessor', 'Not starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
|
||||
}
|
||||
|
||||
callback(null);
|
||||
});
|
||||
} else {
|
||||
log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.stop = (reportId, callback) => {
|
||||
const tid = workers[reportId];
|
||||
if (tid) {
|
||||
log.info('ReportProcessor', 'Killing worker for report id: %s', reportId);
|
||||
executor.stop(tid);
|
||||
reports.updateFields(reportId, { state: reports.ReportState.FAILED}, callback);
|
||||
} else {
|
||||
log.info('ReportProcessor', 'No running worker found for report id: %s', reportId);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.init = callback => {
|
||||
reports.listWithState(reports.ReportState.PROCESSING, 0, 0, (err, reportList) => {
|
||||
if (err) {
|
||||
log.error('ReportProcessor', err);
|
||||
}
|
||||
|
||||
function scheduleReport() {
|
||||
if (reportList.length > 0) {
|
||||
const report = reportList.shift();
|
||||
|
||||
reports.updateFields(report.id, { state: reports.ReportState.SCHEDULED}, err => {
|
||||
if (err) {
|
||||
log.error('ReportProcessor', err);
|
||||
}
|
||||
|
||||
scheduleReport();
|
||||
});
|
||||
}
|
||||
|
||||
startWorkers();
|
||||
return callback();
|
||||
}
|
||||
|
||||
scheduleReport();
|
||||
});
|
||||
};
|
||||
22
lib/tools.js
22
lib/tools.js
|
|
@ -1,5 +1,6 @@
|
|||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
let fs = require('fs');
|
||||
let path = require('path');
|
||||
let db = require('./db');
|
||||
|
|
@ -28,7 +29,6 @@ module.exports = {
|
|||
prepareHtml,
|
||||
purifyHTML,
|
||||
mergeTemplateIntoLayout,
|
||||
nameToFileName,
|
||||
workers: new Set()
|
||||
};
|
||||
|
||||
|
|
@ -130,11 +130,15 @@ function updateMenu(res) {
|
|||
title: _('Automation'),
|
||||
url: '/triggers',
|
||||
key: 'triggers'
|
||||
}, {
|
||||
title: _('Reports'),
|
||||
url: '/reports',
|
||||
key: 'reports'
|
||||
});
|
||||
|
||||
if (config.reports && config.reports.enabled === true) {
|
||||
res.locals.menu.push({
|
||||
title: _('Reports'),
|
||||
url: '/reports',
|
||||
key: 'reports'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function validateEmail(address, checkBlocked, callback) {
|
||||
|
|
@ -302,11 +306,3 @@ function mergeTemplateIntoLayout(template, layout, callback) {
|
|||
}
|
||||
}
|
||||
|
||||
function nameToFileName(name) {
|
||||
return name.
|
||||
trim().
|
||||
toLowerCase().
|
||||
replace(/[ .+/]/g, '-').
|
||||
replace(/[^a-z0-9\-_]/gi, '').
|
||||
replace(/--*/g, '-');
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue