diff --git a/index.js b/index.js index 1ef2340f..57b9d482 100644 --- a/index.js +++ b/index.js @@ -18,6 +18,7 @@ let tzupdate = require('./services/tzupdate'); let feedcheck = require('./services/feedcheck'); let dbcheck = require('./lib/dbcheck'); let tools = require('./lib/tools'); +let reportProcessor = require('./services/report-processor'); let port = config.www.port; let host = config.www.host; @@ -120,23 +121,25 @@ server.on('listening', () => { spawnSenders(() => { feedcheck(() => { postfixBounceServer(() => { - log.info('Service', 'All services started'); - if (config.group) { - try { - process.setgid(config.group); - log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid()); - } catch (E) { - log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message); + reportProcessor.init(() => { + log.info('Service', 'All services started'); + if (config.group) { + try { + process.setgid(config.group); + log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid()); + } catch (E) { + log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message); + } } - } - if (config.user) { - try { - process.setuid(config.user); - log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid()); - } catch (E) { - log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message); + if (config.user) { + try { + process.setuid(config.user); + log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid()); + } catch (E) { + log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message); + } } - } + }); }); }); }); diff --git a/lib/fs-tools.js b/lib/fs-tools.js deleted file mode 100644 index 59f126e6..00000000 --- a/lib/fs-tools.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -module.exports = { - nameToFileName, -}; - -function nameToFileName(name) { - return name. - trim(). - toLowerCase(). - replace(/[ .+/]/g, '-'). - replace(/[^a-z0-9\-_]/gi, ''). - replace(/--*/g, '-'); -} diff --git a/lib/models/campaigns.js b/lib/models/campaigns.js index e9007dcf..1291cd78 100644 --- a/lib/models/campaigns.js +++ b/lib/models/campaigns.js @@ -19,7 +19,7 @@ let tableHelpers = require('../table-helpers'); let allowedKeys = ['description', 'from', 'address', 'reply_to', 'subject', 'editor_name', 'editor_data', 'template', 'source_url', 'list', 'segment', 'html', 'text', 'tracking_disabled']; module.exports.list = (start, limit, callback) => { - tableHelpers.list('campaigns', ['*'], 'scheduled', start, limit, callback); + tableHelpers.list('campaigns', ['*'], 'scheduled', null, start, limit, callback); }; module.exports.filter = (request, parent, callback) => { diff --git a/lib/models/lists.js b/lib/models/lists.js index 772a5820..28304722 100644 --- a/lib/models/lists.js +++ b/lib/models/lists.js @@ -10,7 +10,7 @@ let tableHelpers = require('../table-helpers'); let allowedKeys = ['description', 'default_form', 'public_subscribe']; module.exports.list = (start, limit, callback) => { - tableHelpers.list('lists', ['*'], 'name', start, limit, callback); + tableHelpers.list('lists', ['*'], 'name', null, start, limit, callback); }; module.exports.filter = (request, parent, callback) => { diff --git a/lib/models/report-templates.js b/lib/models/report-templates.js index 195472a3..e43bd7d8 100644 --- a/lib/models/report-templates.js +++ b/lib/models/report-templates.js @@ -8,7 +8,7 @@ const _ = require('../translate')._; const allowedKeys = ['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs']; module.exports.list = (start, limit, callback) => { - tableHelpers.list('report_templates', ['*'], 'name', start, limit, callback); + tableHelpers.list('report_templates', ['*'], 'name', null, start, limit, callback); }; module.exports.quicklist = callback => { diff --git a/lib/models/reports.js b/lib/models/reports.js index 2d972f36..31a53c4f 100644 --- a/lib/models/reports.js +++ b/lib/models/reports.js @@ -8,16 +8,29 @@ const tools = require('../tools'); const _ = require('../translate')._; const log = require('npmlog'); -const allowedKeys = ['name', 'description', 'report_template', 'params', 'filename']; +const allowedKeys = ['name', 'description', 'report_template', 'params']; + +const ReportState = { + SCHEDULED: 0, + PROCESSING: 1, + FINISHED: 2, + FAILED: 3 +}; + +module.exports.ReportState = ReportState; module.exports.list = (start, limit, callback) => { - tableHelpers.list('reports', ['*'], 'name', start, limit, callback); + tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback); }; +module.exports.listWithState = (state, start, limit, callback) => { + tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback); +} + module.exports.filter = (request, callback) => { tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id', - ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.filename AS filename', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ], - request, ['#', 'name', 'report_templates.name', 'description', 'created'], ['name'], 'created DESC', null, callback); + ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ], + request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback); }; module.exports.get = (id, callback) => { @@ -60,29 +73,56 @@ module.exports.get = (id, callback) => { }); }; -module.exports.createOrUpdate = (createMode, data, callback) => { - data = data || {}; +// This method is not supposed to be used for unsanitized inputs. It does not do any checks. +module.exports.updateFields = (id, fieldValueMap, callback) => { + db.getConnection((err, connection) => { + if (err) { + return next(err); + } - const id = 'id' in data ? Number(data.id) : 0; + const clauses = []; + const values = []; + for (let key of Object.keys(fieldValueMap)) { + clauses.push(tools.toDbKey(key) + '=?'); + values.push(fieldValueMap[key]); + } + + values.push(id); + + const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1'; + connection.query(query, values, (err, result) => { + connection.release(); + if (err) { + return callback(err); + } + + return callback(null, result && result.affectedRows || false) + }); + }); +}; + +module.exports.createOrUpdate = (createMode, report, callback) => { + report = report || {}; + + const id = 'id' in report ? Number(report.id) : 0; if (!createMode && id < 1) { return callback(new Error(_('Missing report ID'))); } - const template = tools.convertKeys(data); - const name = (template.name || '').toString().trim(); + const name = (report.name || '').toString().trim(); if (!name) { return callback(new Error(_('Report name must be set'))); } - const reportTemplateId = Number(template.reportTemplate); + const reportTemplateId = Number(report.reportTemplate); reportTemplates.get(reportTemplateId, (err, reportTemplate) => { if (err) { callback(err); } - const params = data.paramsObject; + const params = report.paramsObject; for (const spec of reportTemplate.userFieldsObject) { if (params[spec.id].length < spec.minOccurences) { return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.'))); @@ -97,8 +137,8 @@ module.exports.createOrUpdate = (createMode, data, callback) => { const values = [name, JSON.stringify(params)]; - Object.keys(template).forEach(key => { - let value = typeof template[key] === 'number' ? template[key] : (template[key] || '').toString().trim(); + Object.keys(report).forEach(key => { + let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim(); key = tools.toDbKey(key); if (key === 'description') { diff --git a/lib/models/subscriptions.js b/lib/models/subscriptions.js index f5fb5f84..99d32be0 100644 --- a/lib/models/subscriptions.js +++ b/lib/models/subscriptions.js @@ -21,7 +21,7 @@ module.exports.list = (listId, start, limit, callback) => { return callback(new Error('Missing List ID')); } - tableHelpers.list('subscription__' + listId, ['*'], 'email', start, limit, (err, rows, total) => { + tableHelpers.list('subscription__' + listId, ['*'], 'email', null, start, limit, (err, rows, total) => { if (!err) { rows = rows.map(row => tools.convertKeys(row)); } diff --git a/lib/models/templates.js b/lib/models/templates.js index 09f0abb3..a6723eef 100644 --- a/lib/models/templates.js +++ b/lib/models/templates.js @@ -8,7 +8,7 @@ let tableHelpers = require('../table-helpers'); let allowedKeys = ['description', 'editor_name', 'editor_data', 'html', 'text']; module.exports.list = (start, limit, callback) => { - tableHelpers.list('templates', ['*'], 'name', start, limit, callback); + tableHelpers.list('templates', ['*'], 'name', null, start, limit, callback); }; module.exports.filter = (request, parent, callback) => { diff --git a/lib/table-helpers.js b/lib/table-helpers.js index f5c2e6bc..2c082c2c 100644 --- a/lib/table-helpers.js +++ b/lib/table-helpers.js @@ -4,7 +4,7 @@ let db = require('./db'); let tools = require('./tools'); let log = require('npmlog'); -module.exports.list = (source, fields, orderBy, start, limit, callback) => { +module.exports.list = (source, fields, orderBy, queryData, start, limit, callback) => { db.getConnection((err, connection) => { if (err) { return callback(err); @@ -22,7 +22,15 @@ module.exports.list = (source, fields, orderBy, start, limit, callback) => { } } - connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, limitValues, (err, rows) => { + let whereClause = ''; + let whereValues = []; + + if (queryData) { + whereClause = ' WHERE ' + queryData.where; + whereValues = queryData.values; + } + + connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + whereClause + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, whereValues.concat(limitValues), (err, rows) => { if (err) { connection.release(); return callback(err); diff --git a/lib/tools.js b/lib/tools.js index 080afdcb..7b3074a2 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -28,6 +28,7 @@ module.exports = { prepareHtml, purifyHTML, mergeTemplateIntoLayout, + nameToFileName, workers: new Set() }; @@ -300,3 +301,12 @@ function mergeTemplateIntoLayout(template, layout, callback) { return done(template, layout); } } + +function nameToFileName(name) { + return name. + trim(). + toLowerCase(). + replace(/[ .+/]/g, '-'). + replace(/[^a-z0-9\-_]/gi, ''). + replace(/--*/g, '-'); +} diff --git a/public/css/mailtrain.css b/public/css/mailtrain.css index f8a82266..f4bf6f2f 100644 --- a/public/css/mailtrain.css +++ b/public/css/mailtrain.css @@ -38,4 +38,12 @@ tbody>tr.selected { .table-hover>tbody>tr.selected:hover { background-color: rgb(205, 212, 226); +} + +.row-actions .row-action { + padding-right: 15px; +} + +.row-actions .row-action:last-child { + padding-right: 0px; } \ No newline at end of file diff --git a/public/javascript/tables.js b/public/javascript/tables.js index 3660e2db..be91079e 100644 --- a/public/javascript/tables.js +++ b/public/javascript/tables.js @@ -4,7 +4,62 @@ 'use strict'; -(function(){ +(function() { + function refreshTargets(data) { + for (var target in data) { + var newContent = $(data[target]); + + $(target).replaceWith(newContent); + installHandlers(newContent.parent()); + } + } + + function getAjaxUrl(self) { + var topicId = self.data('topicId'); + var topicUrl = self.data('topicUrl'); + + return topicUrl + '/ajax/' + topicId; + } + + function setupAjaxRefresh() { + var self = $(this); + var ajaxUrl = getAjaxUrl(self); + + var interval = Number(self.data('interval')) || 60; + + setTimeout(function () { + $.get(ajaxUrl, function(data) { + refreshTargets(data); + }); + + }, interval * 1000); + } + + function setupAjaxAction() { + var self = $(this); + var ajaxUrl = getAjaxUrl(self); + + var processing = false; + + self.click(function () { + if (!processing) { + $.get(ajaxUrl, function (data) { + refreshTargets(data); + }); + + processing = true; + } + + return false; + }); + } + + function setupDatestring() { + var self = $(this); + self.html(moment(self.data('date')).fromNow()); + } + + function getDataTableOptions(elem) { var rowSort = $(elem).data('rowSort') || false; @@ -92,6 +147,15 @@ return opts; } + + function installHandlers(elem) { + $('.ajax-refresh', elem).each(setupAjaxRefresh); + $('.ajax-action', elem).each(setupAjaxAction); + $('.datestring', elem).each(setupDatestring); + } + + installHandlers($(document)); + $('.data-table').each(function () { var opts = getDataTableOptions(this); $(this).DataTable(opts); @@ -112,131 +176,135 @@ opts.serverSide = true; opts.processing = true; + opts.createdRow = function( row, data, dataIndex ) { + installHandlers($(row)); + } + $(this).DataTable(opts).on('draw', function () { - $('.datestring').each(function () { - $(this).html(moment($(this).data('date')).fromNow()); - }); + $('.datestring').each(setupDatestring); }); }); + $('.data-stats-pie-chart').each(function () { + var column = $(this).data('column') || 'country'; + var limit = $(this).data('limit') || 20; + var topicId = $(this).data('topicId'); + var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked'; + var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats'; + var self = $(this); + + $.post(ajaxUrl, {column: column, limit: limit}, function(data) { + google.charts.load('current', {'packages':['corechart']}); + google.charts.setOnLoadCallback(drawChart); + + function drawChart() { + var gTable = new google.visualization.DataTable(); + gTable.addColumn('string', 'Column'); + gTable.addColumn('number', 'Value'); + gTable.addRows(data.data); + + var options = {'width':500, 'height':400}; + var chart = new google.visualization.PieChart(self[0]); + chart.draw(gTable, options); + } + }); + }); + + $('.datestring').each(function () { + $(this).html(moment($(this).data('date')).fromNow()); + }); + + $('.delete-form,.confirm-submit').on('submit', function (e) { + if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) { + e.preventDefault(); + } + }); + + $('.fm-date-us.date').datepicker({ + format: 'mm/dd/yyyy', + weekStart: 0, + autoclose: true + }); + + $('.fm-date-eur.date').datepicker({ + format: 'dd/mm/yyyy', + weekStart: 1, + autoclose: true + }); + + $('.fm-date-generic.date').datepicker({ + format: 'yyyy-mm-dd', + weekStart: 1, + autoclose: true + }); + + $('.fm-birthday-us.date').datepicker({ + format: 'mm/dd', + weekStart: 0, + autoclose: true + }); + + $('.fm-birthday-eur.date').datepicker({ + format: 'dd/mm', + weekStart: 1, + autoclose: true + }); + + $('.fm-birthday-generic.date').datepicker({ + format: 'mm-dd', + weekStart: 1, + autoclose: true + }); + + $('.page-refresh').each(function () { + var interval = Number($(this).data('interval')) || 60; + setTimeout(function () { + window.location.reload(); + }, interval * 1000); + }); + + + $('.click-select').on('click', function () { + $(this).select(); + }); + + if (typeof moment.tz !== 'undefined') { + (function () { + var tz = moment.tz.guess(); + if (tz) { + $('.tz-detect').val(tz); + } + })(); + } + + // setup SMTP check + var smtpForm = document.querySelector('form#smtp-verify'); + if (smtpForm) { + smtpForm.addEventListener('submit', function (e) { + e.preventDefault(); + + var form = document.getElementById('settings-form'); + var formData = new FormData(form); + var result = fetch('/settings/smtp-verify', { + method: 'POST', + body: formData, + credentials: 'same-origin' + }); + + var $btn = $('#verify-button').button('loading'); + + result.then(function (res) { + return res.json(); + }).then(function (data) { + alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message); + $btn.button('reset'); + }).catch(function (err) { + alert(err.message); + $btn.button('reset'); + }); + + }); + } + })(); -$('.data-stats-pie-chart').each(function () { - var column = $(this).data('column') || 'country'; - var limit = $(this).data('limit') || 20; - var topicId = $(this).data('topicId'); - var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked'; - var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats'; - var self = $(this); - - $.post(ajaxUrl, {column: column, limit: limit}, function(data) { - google.charts.load('current', {'packages':['corechart']}); - google.charts.setOnLoadCallback(drawChart); - - function drawChart() { - var gTable = new google.visualization.DataTable(); - gTable.addColumn('string', 'Column'); - gTable.addColumn('number', 'Value'); - gTable.addRows(data.data); - - var options = {'width':500, 'height':400}; - var chart = new google.visualization.PieChart(self[0]); - chart.draw(gTable, options); - } - }); -}); - -$('.datestring').each(function () { - $(this).html(moment($(this).data('date')).fromNow()); -}); - -$('.delete-form,.confirm-submit').on('submit', function (e) { - if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) { - e.preventDefault(); - } -}); - -$('.fm-date-us.date').datepicker({ - format: 'mm/dd/yyyy', - weekStart: 0, - autoclose: true -}); - -$('.fm-date-eur.date').datepicker({ - format: 'dd/mm/yyyy', - weekStart: 1, - autoclose: true -}); - -$('.fm-date-generic.date').datepicker({ - format: 'yyyy-mm-dd', - weekStart: 1, - autoclose: true -}); - -$('.fm-birthday-us.date').datepicker({ - format: 'mm/dd', - weekStart: 0, - autoclose: true -}); - -$('.fm-birthday-eur.date').datepicker({ - format: 'dd/mm', - weekStart: 1, - autoclose: true -}); - -$('.fm-birthday-generic.date').datepicker({ - format: 'mm-dd', - weekStart: 1, - autoclose: true -}); - -$('.page-refresh').each(function () { - var interval = Number($(this).data('interval')) || 60; - setTimeout(function () { - window.location.reload(); - }, interval * 1000); -}); - -$('.click-select').on('click', function () { - $(this).select(); -}); - -if (typeof moment.tz !== 'undefined') { - (function () { - var tz = moment.tz.guess(); - if (tz) { - $('.tz-detect').val(tz); - } - })(); -} - -// setup SMTP check -var smtpForm = document.querySelector('form#smtp-verify'); -if (smtpForm) { - smtpForm.addEventListener('submit', function (e) { - e.preventDefault(); - - var form = document.getElementById('settings-form'); - var formData = new FormData(form); - var result = fetch('/settings/smtp-verify', { - method: 'POST', - body: formData, - credentials: 'same-origin' - }); - - var $btn = $('#verify-button').button('loading'); - - result.then(function (res) { - return res.json(); - }).then(function (data) { - alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message); - $btn.button('reset'); - }).catch(function (err) { - alert(err.message); - $btn.button('reset'); - }); - - }); -} diff --git a/routes/report-templates.js b/routes/report-templates.js index 638502fd..5393ccb4 100644 --- a/routes/report-templates.js +++ b/routes/report-templates.js @@ -93,7 +93,7 @@ router.get('/create', passport.csrfProtection, (req, res) => { '

{{title}}

\n' + '\n' + '
\n' + - ' \n' + + '
\n' + ' \n' + '
\n' + ' {{#translate}}Email{{/translate}}\n' + diff --git a/routes/reports.js b/routes/reports.js index 5006668c..af15abce 100644 --- a/routes/reports.js +++ b/routes/reports.js @@ -6,6 +6,7 @@ const router = new express.Router(); const _ = require('../lib/translate')._; const reportTemplates = require('../lib/models/report-templates'); const reports = require('../lib/models/reports'); +const reportProcessor = require('../services/report-processor'); const campaigns = require('../lib/models/campaigns'); const lists = require('../lib/models/lists'); const tools = require('../lib/tools'); @@ -13,7 +14,7 @@ const util = require('util'); const htmlescape = require('escape-html'); const striptags = require('striptags'); const fs = require('fs'); -const fsTools = require('../lib/fs-tools'); +const hbs = require('hbs'); router.all('/*', (req, res, next) => { if (!req.user) { @@ -30,26 +31,9 @@ router.get('/', (req, res) => { }); }); + + router.post('/ajax', (req, res) => { - function getViewLink(row) { - if (row.state == 0) { - // TODO: Render waiting - // TODO: Add error output - return ' '; - } else if (row.state == 1) { - let icon = 'eye-open'; - if (row.mimeType == 'text/csv') icon = 'download-alt'; - - // TODO: Add error output - return ' '; - } else if (row.state == 2) { - // TODO: Add error output - return ' '; - } - - return ''; - } - reports.filter(req.body, (err, data, total, filteredTotal) => { if (err) { return res.json({ @@ -67,14 +51,29 @@ router.post('/ajax', (req, res) => { htmlescape(row.name || ''), htmlescape(row.reportTemplateName || ''), htmlescape(striptags(row.description) || ''), - '' + row.created.toISOString() + '', - getViewLink(row) + - ''] - ) + getRowLastRun(row), + getRowActions(row) + ]) }); }); }); +router.get('/row/ajax/:id', (req, res) => { + respondRowActions(req.params.id, res); +}); + +router.get('/start/ajax/:id', (req, res) => { + reportProcessor.start(req.params.id, () => { + respondRowActions(req.params.id, res); + }); +}); + +router.get('/stop/ajax/:id', (req, res) => { + reportProcessor.stop(req.params.id, () => { + respondRowActions(req.params.id, res); + }); +}); + router.get('/create', passport.csrfProtection, (req, res) => { const reqData = req.query; reqData.csrfToken = req.csrfToken(); @@ -116,7 +115,6 @@ router.get('/create', passport.csrfProtection, (req, res) => { router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => { const reqData = req.body; - delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report. const reportTemplateId = Number(reqData.reportTemplate); @@ -131,6 +129,9 @@ router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) = req.flash('danger', err && err.message || err || _('Could not create report')); return res.redirect('/reports/create?' + tools.queryParams(data)); } + + reportProcessor.start(id); + req.flash('success', util.format(_('Report “%s” created'), data.name)); res.redirect('/reports'); }); @@ -179,8 +180,6 @@ router.get('/edit/:id', passport.csrfProtection, (req, res) => { router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => { const reqData = req.body; - delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report. - const reportTemplateId = Number(reqData.reportTemplate); addParamsObject(reportTemplateId, reqData, (err, data) => { @@ -231,10 +230,10 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => { return res.redirect('/reports'); } - if (report.state == 1) { + if (report.state == reports.ReportState.FINISHED) { if (reportTemplate.mimeType == 'text/html') { - fs.readFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), (err, reportContent) => { + fs.readFile(reportProcessor.getFileName(report, 'report'), (err, reportContent) => { if (err) { req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); return res.redirect('/reports'); @@ -251,11 +250,11 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => { } else if (reportTemplate.mimeType == 'text/csv') { const headers = { - 'Content-Disposition': 'attachment;filename=' + fsTools.nameToFileName(report.name) + '.csv', + 'Content-Disposition': 'attachment;filename=' + tools.nameToFileName(report.name) + '.csv', 'Content-Type': 'text/csv' }; - res.sendFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), {headers: headers}); + res.sendFile(reportProcessor.getFileName(report, 'report'), {headers: headers}); } else { req.flash('danger', _('Unknown type of template')); @@ -270,6 +269,83 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => { }); }); +router.get('/output/:id', passport.csrfProtection, (req, res) => { + reports.get(req.params.id, (err, report) => { + if (err || !report) { + req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); + return res.redirect('/reports'); + } + + fs.readFile(reportProcessor.getFileName(report, 'output'), (err, output) => { + let data = { + csrfToken: req.csrfToken(), + title: 'Output for report ' + report.name + }; + + if (err) { + data.error = 'No output.'; + } else { + data.output = output; + } + + res.render('reports/output', data); + }); + }); +}); + +function getRowLastRun(row) { + return '' + (row.lastRun ? '' + row.lastRun.toISOString() + '' : '') + ''; +} + +function getRowActions(row) { + let requestRefresh = false; + let view, startStop; + let topic = 'data-topic-id="' + row.id + '"'; + + if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) { + view = ''; + startStop = ''; + requestRefresh = true; + + } else if (row.state == reports.ReportState.FINISHED) { + let icon = 'eye-open'; + if (row.mimeType == 'text/csv') icon = 'download-alt'; + + view = ''; + startStop = ''; + + } else if (row.state == reports.ReportState.FAILED) { + view = ''; + startStop = ''; + } + + let actions = view; + actions += ''; + actions += startStop; + actions += ''; + + return '' + + actions + + ''; +} + +function respondRowActions(id, res) { + reports.get(id, (err, report) => { + if (err) { + return res.json({ + error: err, + }); + } + + const data = {}; + data['#row-last-run-' + id] = getRowLastRun(report); + data['#row-actions-' + id] = getRowActions(report); + + res.json(data); + }); +} + + function addUserFields(reportTemplateId, reqData, report, callback) { reportTemplates.get(reportTemplateId, (err, reportTemplate) => { if (err) { diff --git a/services/reports.js b/services/report-processor-worker.js similarity index 80% rename from services/reports.js rename to services/report-processor-worker.js index 866badc8..d4141bd6 100644 --- a/services/reports.js +++ b/services/report-processor-worker.js @@ -11,8 +11,7 @@ const hbs = require('hbs'); const vm = require('vm'); const log = require('npmlog'); const fs = require('fs'); -const path = require('path'); -const fsTools = require('../lib/fs-tools'); +const reportProcessor = require('./report-processor'); handlebarsHelpers.registerHelpers(handlebars); @@ -74,64 +73,52 @@ function resolveUserFields(userFields, params, callback) { } function doneSuccess(id) { - // TODO: Mark in the DB as completed + update the date/time - // TODO update the report filename in the DB - process.exit(0); } function doneFail(id) { - // TODO: Mark in the DB as failed process.exit(1); } -function start(id) { - // TODO: Mark in the DB as running -} - -// TODO: Retrieve report task from the DB and run it - function processReport(reportId) { reports.get(reportId, (err, report) => { if (err || !report) { log.error('reports', err && err.message || err || _('Could not find report with specified ID')); - doneFail(reportId); + doneFail(); } reportTemplates.get(report.reportTemplate, (err, reportTemplate) => { if (err) { log.error('reports', err && err.message || err || _('Could not find report template')); - doneFail(reportId); + doneFail(); } resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => { if (err) { log.error('reports', err.message || err); - doneFail(reportId); + doneFail(); } - const filename = fsTools.nameToFileName(report.name); - const sandbox = { require: require, inputs: inputs, + console: console, callback: (err, outputs) => { if (err) { log.error('reports', err.message || err); - doneFail(reportId); + doneFail(); } const hbsTmpl = handlebars.compile(reportTemplate.hbs); const reportText = hbsTmpl(outputs); - fs.writeFile(path.join(__dirname, '../protected/reports', filename + '.report'), reportText, (err, reportContent) => { + fs.writeFile(reportProcessor.getFileName(report, 'report'), reportText, (err, reportContent) => { if (err) { log.error('reports', err && err.message || err || _('Could not find report with specified ID')); - doneFail(reportId); + doneFail(); } - doneSuccess(reportId, filename); - process + doneSuccess(); }); } }; @@ -143,4 +130,4 @@ function processReport(reportId) { }); } -processReport(1); +processReport(Number(process.argv[2])); diff --git a/services/report-processor.js b/services/report-processor.js new file mode 100644 index 00000000..be6cadf1 --- /dev/null +++ b/services/report-processor.js @@ -0,0 +1,154 @@ +'use strict'; + +const log = require('npmlog'); +const db = require('../lib/db'); +const reports = require('../lib/models/reports'); +const _ = require('../lib/translate')._; +const path = require('path'); +const tools = require('../lib/tools'); +const fs = require('fs'); +const fork = require('child_process').fork; + +let runningWorkersCount = 0; +let maxWorkersCount = 1; + +let workers = {}; + +function getFileName(report, suffix) { + return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + tools.nameToFileName(report.name) + '.' + suffix); +} + +module.exports.getFileName = getFileName; + +function spawnWorker(report) { + + fs.open(getFileName(report, 'output'), 'w', (err, outFd) => { + if (err) { + log.error('ReportProcessor', err); + return; + } + + runningWorkersCount++; + + const options = { + stdio: ['ignore', outFd, outFd, 'ipc'], + cwd: path.join(__dirname, '..') + }; + + let child = fork(path.join(__dirname, 'report-processor-worker.js'), [report.id], options); + let pid = child.pid; + workers[report.id] = child; + + log.info('ReportProcessor', 'Worker process for "%s" started with pid %s. Current worker count is %s.', report.name, pid, runningWorkersCount); + + child.on('close', (code, signal) => { + runningWorkersCount--; + + delete workers[report.id]; + log.info('ReportProcessor', 'Worker process for "%s" (pid %s) exited with code %s signal %s. Current worker count is %s.', report.name, pid, code, signal, runningWorkersCount); + + fs.close(outFd, (err) => { + if (err) { + log.error('ReportProcessor', err); + } + + const fields = {}; + if (code ===0 ) { + fields.state = reports.ReportState.FINISHED; + fields.lastRun = new Date(); + } else { + fields.state = reports.ReportState.FAILED; + } + + reports.updateFields(report.id, fields, (err) => { + if (err) { + log.error('ReportProcessor', err); + } + + setImmediate(worker); + }); + }); + }); + }); +}; + +function worker() { + reports.listWithState(reports.ReportState.SCHEDULED, 0, maxWorkersCount - runningWorkersCount, (err, reportList) => { + if (err) { + log.error('ReportProcessor', err); + return; + } + + for (let report of reportList) { + reports.updateFields(report.id, { state: reports.ReportState.PROCESSING }, (err) => { + if (err) { + log.error('ReportProcessor', err); + return; + } + + spawnWorker(report); + }); + } + }); +} + +module.exports.start = (reportId, callback) => { + if (!workers[reportId]) { + log.info('ReportProcessor', 'Scheduling report id: %s', reportId); + reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}, (err) => { + if (err) { + return callback(err); + } + + if (runningWorkersCount < maxWorkersCount) { + log.info('ReportProcessor', 'Starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount); + + worker(); + } else { + log.info('ReportProcessor', 'Not starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount); + } + + callback(null); + }); + } else { + log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId); + } +}; + +module.exports.stop = (reportId, callback) => { + const child = workers[reportId]; + if (child) { + log.info('ReportProcessor', 'Killing worker for report id: %s', reportId); + child.kill(); + reports.updateFields(reportId, { state: reports.ReportState.FAILED}, callback); + } else { + log.info('ReportProcessor', 'No running worker found for report id: %s', reportId); + } +}; + +module.exports.init = (callback) => { + reports.listWithState(reports.ReportState.PROCESSING, 0, 0, (err, reportList) => { + if (err) { + log.error('ReportProcessor', err); + } + + function scheduleReport() { + if (reportList.length > 0) { + const report = reportList.shift(); + + reports.updateFields(report.id, { state: reports.ReportState.SCHEDULED}, (err) => { + if (err) { + log.error('ReportProcessor', err); + } + + scheduleReport(); + }); + } + + worker(); + callback(); + } + + scheduleReport(); + }); +}; diff --git a/setup/sql/upgrade-00027.sql b/setup/sql/upgrade-00027.sql index 2721dcdc..0f098534 100644 --- a/setup/sql/upgrade-00027.sql +++ b/setup/sql/upgrade-00027.sql @@ -22,7 +22,6 @@ CREATE TABLE `reports` ( `description` text, `report_template` int(11) unsigned NOT NULL, `params` longtext, - `filename` varchar(255) DEFAULT NULL, `state` int(11) unsigned NOT NULL DEFAULT 0, `last_run` DATETIME DEFAULT NULL, `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/views/campaigns/campaigns.hbs b/views/campaigns/campaigns.hbs index 9ac302ea..39a01fb6 100644 --- a/views/campaigns/campaigns.hbs +++ b/views/campaigns/campaigns.hbs @@ -23,7 +23,7 @@
- - diff --git a/views/lists/lists.hbs b/views/lists/lists.hbs index 93cc4918..82a935d1 100644 --- a/views/lists/lists.hbs +++ b/views/lists/lists.hbs @@ -14,7 +14,7 @@
+ # @@ -38,7 +38,7 @@ {{#translate}}Created{{/translate}} +  
- - diff --git a/views/reports/output.hbs b/views/reports/output.hbs new file mode 100644 index 00000000..3da8d65f --- /dev/null +++ b/views/reports/output.hbs @@ -0,0 +1,8 @@ +
    +
  1. {{#translate}}Home{{/translate}}
    2. +
  2. {{#translate}}Reports{{/translate}}
    3. +
  3. {{title}}
    4. +
+ +{{error}} +
{{output}}
diff --git a/views/reports/reports.hbs b/views/reports/reports.hbs index 6998a8d2..99d2c40b 100644 --- a/views/reports/reports.hbs +++ b/views/reports/reports.hbs @@ -16,7 +16,7 @@
+ # @@ -29,7 +29,7 @@ {{#translate}}Description{{/translate}} +  
- -
+ # @@ -31,9 +31,10 @@ {{#translate}}Created{{/translate}} +  
+ diff --git a/views/templates/templates.hbs b/views/templates/templates.hbs index 0314eb3e..e88aac21 100644 --- a/views/templates/templates.hbs +++ b/views/templates/templates.hbs @@ -14,7 +14,7 @@
- -
+ # @@ -23,7 +23,7 @@ {{#translate}}Description{{/translate}} +