public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Reports ported to ReactJS and Knex

Browse source 
Note that the interface for the custom JS code inside a report template has changed. It now offers promise-based interface and exposes knex.
This commit is contained in:
Tomas Bures 2017-07-13 13:27:03 +02:00
parent 6d95fa515e
commit d63eed9ca9
27 changed files with 649 additions and 953 deletions
Download patch file Download diff file Expand all files Collapse all files

41
app.js
View file

@ -40,6 +40,9 @@ const editorapi = require('./routes/editorapi');
const grapejs = require('./routes/grapejs');
const mosaico = require('./routes/mosaico');
// These are routes for the new React-based client
const reports = require('./routes/reports');
const namespacesRest = require('./routes/rest/namespaces');
const usersRest = require('./routes/rest/users');
const accountRest = require('./routes/rest/account');
@ -242,18 +245,22 @@ app.use('/editorapi', editorapi);
app.use('/grapejs', grapejs);
app.use('/mosaico', mosaico);
if (config.reports && config.reports.enabled === true) {
app.use('/reports', reports);
}
/* FIXME - this should be removed once we bind the ReactJS client to / */
app.use('/users', usersLegacyIntegration);
app.use('/namespaces', namespacesLegacyIntegration);
app.use('/account', accountLegacyIntegration);
if (config.reports && config.reports.enabled === true) {
app.use('/reports', reports);
app.use('/reports', reportsLegacyIntegration);
}
/* ------------------------------------------------------------------- */
app.all('/rest/*', (req, res, next) => {
req.needsJSONResponse = true;
next();
@ -301,11 +308,16 @@ if (app.get('env') === 'development') {
res.status(err.status || 500).json(resp);
} else {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
if (err instanceof interoperableErrors.NotLoggedInError) {
req.flash('danger', _('Need to be logged in to access restricted content'));
return res.redirect('/account/login?next=' + encodeURIComponent(req.originalUrl));
} else {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
}
}
});
@ -331,11 +343,16 @@ if (app.get('env') === 'development') {
res.status(err.status || 500).json(resp);
} else {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
if (err instanceof interoperableErrors.NotLoggedInError) {
req.flash('danger', _('Need to be logged in to access restricted content'));
return res.redirect('/account/login?next=' + encodeURIComponent(req.originalUrl));
} else {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
}
}
});
}

64
client/src/lib/form.js
View file

@ -49,36 +49,6 @@ class Form extends Component {
};
}
static async handleChangedError(owner, fn) {
try {
await fn();
} catch (error) {
if (error instanceof interoperableErrors.ChangedError) {
owner.disableForm();
owner.setFormStatusMessage('danger',
<span>
<strong>{t('Your updates cannot be saved.')}</strong>{' '}
{t('Someone else has introduced modification in the meantime. Refresh your page to start anew with fresh data. Please note that your changes will be lost.')}
</span>
);
return;
}
if (error instanceof interoperableErrors.NotFoundError) {
owner.disableForm();
owner.setFormStatusMessage('danger',
<span>
<strong>{t('Your updates cannot be saved.')}</strong>{' '}
{t('It seems that someone else has deleted the entity in the meantime.')}
</span>
);
return;
}
throw error;
}
}
@withAsyncErrorHandler
async onSubmit(evt) {
const t = this.props.t;
@ -88,7 +58,7 @@ class Form extends Component {
evt.preventDefault();
if (this.props.onSubmitAsync) {
await Form.handleChangedError(owner, async () => await this.props.onSubmitAsync(evt));
await this.formHandleChangedError(async () => await this.props.onSubmitAsync(evt));
}
}
@ -520,6 +490,7 @@ class ACEEditor extends Component {
showPrintMargin={false}
value={owner.getFormValue(id)}
tabSize={2}
setOptions={{useWorker: false}} // This disables syntax check because it does not always work well (e.g. in case of JS code in report templates)
/>
);
}
@ -839,6 +810,37 @@ function withForm(target) {
return this.state.formState.get('isDisabled');
};
inst.formHandleChangedError = async function(fn) {
const t = this.props.t;
try {
await fn();
} catch (error) {
if (error instanceof interoperableErrors.ChangedError) {
this.disableForm();
this.setFormStatusMessage('danger',
<span>
<strong>{t('Your updates cannot be saved.')}</strong>{' '}
{t('Someone else has introduced modification in the meantime. Refresh your page to start anew with fresh data. Please note that your changes will be lost.')}
</span>
);
return;
}
if (error instanceof interoperableErrors.NotFoundError) {
this.disableForm();
this.setFormStatusMessage('danger',
<span>
<strong>{t('Your updates cannot be saved.')}</strong>{' '}
{t('It seems that someone else has deleted the entity in the meantime.')}
</span>
);
return;
}
throw error;
}
};
return target;
}

4
client/src/lib/page.css
View file

@ -11,11 +11,11 @@
padding-bottom: 5px;
}
.mt-action-links > a {
.mt-action-links > * {
margin-right: 8px;
}
.mt-action-links > a:last-child {
.mt-action-links > *:last-child {
margin-right: 0px;
}

87
client/src/lib/table.js
View file

@ -29,7 +29,7 @@ const TableSelectMode = {
};
@translate()
@translate(null, { withRef: true })
@withPageHelpers
@withErrorHandling
class Table extends Component {
@ -48,8 +48,9 @@ class Table extends Component {
selectionAsArray: PropTypes.bool,
onSelectionChangedAsync: PropTypes.func,
onSelectionDataAsync: PropTypes.func,
actionLinks: PropTypes.array,
withHeader: PropTypes.bool
actions: PropTypes.func,
withHeader: PropTypes.bool,
refreshInterval: PropTypes.number
}
static defaultProps = {
@ -57,6 +58,12 @@ class Table extends Component {
selectionKeyIndex: 0
}
refresh() {
if (this.table) {
this.table.rows().draw('page');
}
}
getSelectionMap(props) {
let selArray = [];
if (props.selectMode === TableSelectMode.SINGLE && !this.props.selectionAsArray) {
@ -162,17 +169,56 @@ class Table extends Component {
componentDidMount() {
const columns = this.props.columns.slice();
if (this.props.actionLinks) {
const actionLinks = this.props.actionLinks;
if (this.props.actions) {
const createdCellFn = (td, data) => {
const linksContainer = jQuery('<span class="mt-action-links"/>');
for (const {label, link} of actionLinks) {
const dest = link(data);
const lnkHtml = ReactDOMServer.renderToStaticMarkup(<a href={dest}>{label}</a>);
const lnk = jQuery(lnkHtml);
lnk.click((evt) => { evt.preventDefault(); this.navigateTo(dest) });
linksContainer.append(lnk);
let actions = this.props.actions(data);
let options = {};
if (!Array.isArray(actions)) {
options = actions;
actions = actions.actions;
}
for (const action of actions) {
if (action.action) {
const html = ReactDOMServer.renderToStaticMarkup(<a href="">{action.label}</a>);
const elem = jQuery(html);
elem.click((evt) => { evt.preventDefault(); action.action(this) });
linksContainer.append(elem);
} else if (action.link) {
const html = ReactDOMServer.renderToStaticMarkup(<a href={action.link}>{action.label}</a>);
const elem = jQuery(html);
elem.click((evt) => { evt.preventDefault(); this.navigateTo(action.link) });
linksContainer.append(elem);
} else if (action.href) {
const html = ReactDOMServer.renderToStaticMarkup(<a href={action.href}>{action.label}</a>);
const elem = jQuery(html);
linksContainer.append(elem);
} else {
const html = ReactDOMServer.renderToStaticMarkup(action.label);
const elem = jQuery(html);
linksContainer.append(elem);
}
}
if (options.refreshTimeout) {
const currentMS = Date.now();
if (!this.refreshTimeoutAt || this.refreshTimeoutAt > currentMS + options.refreshTimeout) {
clearTimeout(this.refreshTimeoutId);
this.refreshTimeoutAt = currentMS + options.refreshTimeout;
this.refreshTimeoutId = setTimeout(() => {
this.refreshTimeoutAt = 0;
this.refresh();
}, options.refreshTimeout);
}
}
jQuery(td).html(linksContainer);
@ -238,6 +284,15 @@ class Table extends Component {
this.table = jQuery(this.domTable).DataTable(dtOptions);
if (this.props.refreshInterval) {
this.refreshIntervalId = setInterval(() => this.refresh(), this.props.refreshInterval);
}
this.table.on('destroy.dt', () => {
clearInterval(this.refreshIntervalId);
clearTimeout(this.refreshTimeoutId);
});
this.fetchSelectionData();
}
@ -306,6 +361,14 @@ class Table extends Component {
}
}
/*
Refreshes the table. This method is provided to allow programmatic refresh from a handler outside the table.
The reference to the table can be obtained by ref.
*/
Table.prototype.refresh = function() {
this.getWrappedInstance().refresh()
};
export {
Table,
TableSelectMode

21
client/src/lib/tree.js
View file

@ -65,7 +65,7 @@ class TreeTable extends Component {
selectMode: PropTypes.number,
selection: PropTypes.oneOfType([PropTypes.array, PropTypes.string, PropTypes.number]),
onSelectionChangedAsync: PropTypes.func,
actionLinks: PropTypes.array,
actions: PropTypes.func,
withHeader: PropTypes.bool
}
@ -100,19 +100,18 @@ class TreeTable extends Component {
};
let createNodeFn;
if (this.props.actionLinks) {
const actionLinks = this.props.actionLinks;
if (this.props.actions) {
createNodeFn = (event, data) => {
const node = data.node;
const tdList = jQuery(node.tr).find(">td");
const linksContainer = jQuery('<span class="mt-action-links"/>');
for (const {label, link} of actionLinks) {
const dest = link(node.key);
const lnkHtml = ReactDOMServer.renderToStaticMarkup(<a href={dest}>{label}</a>);
const actions = this.props.actions(node.key);
for (const {label, link} of actions) {
const lnkHtml = ReactDOMServer.renderToStaticMarkup(<a href={link}>{label}</a>);
const lnk = jQuery(lnkHtml);
lnk.click((evt) => { evt.preventDefault(); this.navigateTo(dest) });
lnk.click((evt) => { evt.preventDefault(); this.navigateTo(link) });
linksContainer.append(lnk);
}
@ -202,7 +201,7 @@ class TreeTable extends Component {
render() {
const t = this.props.t;
const props = this.props;
const actionLinks = props.actionLinks;
const actions = props.actions;
const withHeader = props.withHeader;
let containerClass = 'mt-treetable-container';
@ -223,14 +222,14 @@ class TreeTable extends Component {
<thead>
<tr>
<th>{t('Name')}</th>
{actionLinks && <th></th>}
{actions && <th></th>}
</tr>
</thead>
}
<tbody>
<tr>
<td></td>
{actionLinks && <td></td>}
{actions && <td></td>}
</tr>
</tbody>
</table>

6
client/src/namespaces/List.js
View file

@ -10,10 +10,10 @@ export default class List extends Component {
render() {
const t = this.props.t;
const actionLinks = [
const actions = key => [
{
label: 'Edit',
link: key => '/namespaces/edit/' + key
link: '/namespaces/edit/' + key
}
];
@ -25,7 +25,7 @@ export default class List extends Component {
<Title>{t('Namespaces')}</Title>
<TreeTable withHeader dataUrl="/rest/namespaces-tree" actionLinks={actionLinks} />
<TreeTable withHeader dataUrl="/rest/namespaces-tree" actions={actions} />
</div>
);
}

88
client/src/reports/List.js
View file

@ -4,24 +4,100 @@ import React, { Component } from 'react';
import { translate } from 'react-i18next';
import { Title, Toolbar, NavButton } from '../lib/page';
import { Table } from '../lib/table';
import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
import moment from 'moment';
import axios from '../lib/axios';
import { ReportState } from '../../../shared/reports';
@translate()
@withErrorHandling
export default class List extends Component {
@withAsyncErrorHandler
async stop(table, id) {
await axios.post(`/rest/report-stop/${id}`);
table.refresh();
}
@withAsyncErrorHandler
async start(table, id) {
await axios.post(`/rest/report-start/${id}`);
table.refresh();
}
render() {
const t = this.props.t;
const actionLinks = [{
label: 'Edit',
link: data => '/reports/edit/' + data[0]
}];
const actions = data => {
let view, startStop, refreshTimeout;
const state = data[5];
const id = data[0];
const mimeType = data[6];
if (state === ReportState.PROCESSING || state === ReportState.SCHEDULED) {
view = {
label: <span className="glyphicon glyphicon-hourglass" aria-hidden="true" title="Processing"></span>,
};
startStop = {
label: <span className="glyphicon glyphicon-stop" aria-hidden="true" title="Stop"></span>,
action: (table) => this.stop(table, id)
};
refreshTimeout = 1000;
} else if (state === ReportState.FINISHED) {
if (mimeType === 'text/html') {
view = {
label: <span className="glyphicon glyphicon-eye-open" aria-hidden="true" title="View"></span>,
link: `reports/view/${id}`
};
} else if (mimeType === 'text/csv') {
view = {
label: <span className="glyphicon glyphicon-download-alt" aria-hidden="true" title="Download"></span>,
href: `reports/download/${id}`
};
}
startStop = {
label: <span className="glyphicon glyphicon-repeat" aria-hidden="true" title="Refresh report"></span>,
action: (table) => this.start(table, id)
};
} else if (state === ReportState.FAILED) {
view = {
label: <span className="glyphicon glyphicon-thumbs-down" aria-hidden="true" title="Report generation failed"></span>,
};
startStop = {
label: <span className="glyphicon glyphicon-repeat" aria-hidden="true" title="Regenerate report"></span>,
action: (table) => this.start(table, id)
};
}
return {
refreshTimeout,
actions: [
view,
{
label: <span className="glyphicon glyphicon-modal-window" aria-hidden="true" title="View console output"></span>,
link: `reports/output/${id}`
},
startStop,
{
label: <span className="glyphicon glyphicon-wrench" aria-hidden="true" title="Edit"></span>,
link: `/reports/edit/${id}`
}
]
};
};
const columns = [
{ data: 0, title: "#" },
{ data: 1, title: t('Name') },
{ data: 2, title: t('Template') },
{ data: 3, title: t('Description') },
{ data: 4, title: t('Last Run'), render: data => data ? moment(data).fromNow() : t('Not run yet') }
{ data: 4, title: t('Created'), render: data => data ? moment(data).fromNow() : '' }
];
return (
@ -33,7 +109,7 @@ export default class List extends Component {
<Title>{t('Reports')}</Title>
<Table withHeader dataUrl="/rest/reports-table" columns={columns} actionLinks={actionLinks} />
<Table withHeader dataUrl="/rest/reports-table" columns={columns} actions={actions} />
</div>
);
}

53
client/src/reports/Output.js Normal file
View file

@ -0,0 +1,53 @@
'use strict';
import React, { Component } from 'react';
import { translate } from 'react-i18next';
import { withPageHelpers, Title } from '../lib/page'
import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
import axios from '../lib/axios';
@translate()
@withPageHelpers
@withErrorHandling
export default class Output extends Component {
constructor(props) {
super(props);
this.state = {
output: null
};
}
@withAsyncErrorHandler
async loadOutput() {
const id = parseInt(this.props.match.params.id);
const outputResp = await axios.get(`/rest/report-output/${id}`);
const reportResp = await axios.get(`/rest/reports/${id}`);
this.setState({
output: outputResp.data,
report: reportResp.data,
});
}
componentDidMount() {
this.loadOutput();
}
render() {
const t = this.props.t;
if (this.state.report) {
return (
<div>
<Title>{t('Output for report {{name}}', { name: this.state.report.name })}</Title>
<pre>{this.state.output}</pre>
</div>
);
} else {
return <div>{t('Loading report output ...')}</div>;
}
}
}

57
client/src/reports/View.js Normal file
View file

@ -0,0 +1,57 @@
'use strict';
import React, { Component } from 'react';
import { translate } from 'react-i18next';
import { withPageHelpers, Title } from '../lib/page'
import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
import axios from '../lib/axios';
import { ReportState } from '../../../shared/reports';
@translate()
@withPageHelpers
@withErrorHandling
export default class View extends Component {
constructor(props) {
super(props);
this.state = {
content: null
};
}
@withAsyncErrorHandler
async loadContent() {
const id = parseInt(this.props.match.params.id);
const contentResp = await axios.get(`/rest/report-content/${id}`);
const reportResp = await axios.get(`/rest/reports/${id}`);
this.setState({
content: contentResp.data,
report: reportResp.data
});
}
componentDidMount() {
this.loadContent();
}
render() {
const t = this.props.t;
if (this.state.report) {
if (this.state.report.state === ReportState.FINISHED) {
return (
<div>
<Title>{t('Report {{name}}', { name: this.state.report.name })}</Title>
<div dangerouslySetInnerHTML={{ __html: this.state.content }}/>
</div>
);
} else {
return <div className="alert alert-danger" role="alert">{t('Report not generated')}</div>;
}
} else {
return <div>{t('Loading report ...')}</div>;
}
}
}

12
client/src/reports/root.js
View file

@ -8,6 +8,8 @@ import i18n from '../lib/i18n';
import { Section } from '../lib/page'
import ReportsCUD from './CUD'
import ReportsList from './List'
import ReportsView from './View'
import ReportsOutput from './Output'
import ReportTemplatesCUD from './templates/CUD'
import ReportTemplatesList from './templates/List'
@ -33,6 +35,16 @@ const getStructure = t => {
title: t('Create Report'),
render: props => (<ReportsCUD {...props} />)
},
view: {
title: t('View Report'),
params: [':id' ],
render: props => (<ReportsView {...props} />)
},
output: {
title: t('View Report Output'),
params: [':id' ],
render: props => (<ReportsOutput {...props} />)
},
'templates': {
title: t('Templates'),
link: '/reports/templates',

53
client/src/reports/templates/CUD.js
View file

@ -57,17 +57,8 @@ export default class CUD extends Component {
' }\n' +
']',
js:
'campaigns.results(inputs.campaign, ["*"], "", (err, results) => {\n' +
' if (err) {\n' +
' return callback(err);\n' +
' }\n' +
'\n' +
' const data = {\n' +
' results: results\n' +
' };\n' +
'\n' +
' return callback(null, data);\n' +
'});',
'const results = await campaigns.getResults(inputs.campaign, ["*"]);\n' +
'render({ results });',
hbs:
'<h2>{{title}}</h2>\n' +
'\n' +
@ -115,21 +106,17 @@ export default class CUD extends Component {
' }\n' +
']',
js:
'campaigns.results(inputs.campaign, ["custom_country", "count(*) AS count_all", "SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"], "GROUP BY custom_country", (err, results) => {\n' +
' if (err) {\n' +
' return callback(err);\n' +
' }\n' +
'const results = await campaigns.getResults(inputs.campaign, ["custom_country"], query =>\n' +
' query.count("* AS count_all")\n' +
' .select(knex.raw("SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"))\n' +
' .groupBy("custom_country")\n' +
');\n' +
'\n' +
' for (let row of results) {\n' +
' row["percentage"] = Math.round((row.count_opened / row.count_all) * 100);\n' +
' }\n' +
'for (const row of results) {\n' +
' row.percentage = Math.round((row.count_opened / row.count_all) * 100);\n' +
'}\n' +
'\n' +
' let data = {\n' +
' results: results\n' +
' };\n' +
'\n' +
' return callback(null, data);\n' +
'});',
'render({ results });',
hbs:
'<h2>{{title}}</h2>\n' +
'\n' +
@ -189,17 +176,8 @@ export default class CUD extends Component {
' }\n' +
']',
js:
'subscriptions.list(inputs.list.id,0,0, (err, results) => {\n' +
' if (err) {\n' +
' return callback(err);\n' +
' }\n' +
'\n' +
' let data = {\n' +
' results: results\n' +
' };\n' +
'\n' +
' return callback(null, data);\n' +
'});',
'const results = await subscriptions.list(inputs.list.id);\n' +
'render({ results });',
hbs:
'{{#each results}}\n' +
'{{firstName}},{{lastName}},{{email}}\n' +
@ -246,11 +224,11 @@ export default class CUD extends Component {
}
async submitAndStay() {
await Form.handleChangedError(this, async () => await this.doSubmit(true));
await this.formHandleChangedError(async () => await this.doSubmit(true));
}
async submitAndLeave() {
await Form.handleChangedError(this, async () => await this.doSubmit(false));
await this.formHandleChangedError(async () => await this.doSubmit(false));
}
async doSubmit(stay) {
@ -273,6 +251,7 @@ export default class CUD extends Component {
if (submitSuccessful) {
if (stay) {
await this.loadFormValues();
this.enableForm();
this.setFormStatusMessage('success', t('Report template saved'));
} else {

12
client/src/reports/templates/List.js
View file

@ -12,10 +12,12 @@ export default class List extends Component {
render() {
const t = this.props.t;
const actionLinks = [{
label: 'Edit',
link: data => '/reports/templates/edit/' + data[0]
}];
const actions = data => [
{
label: 'Edit',
link: '/reports/templates/edit/' + data[0]
}
];
const columns = [
{ data: 0, title: "#" },
@ -37,7 +39,7 @@ export default class List extends Component {
<Title>{t('Report Templates')}</Title>
<Table withHeader dataUrl="/rest/report-templates-table" columns={columns} actionLinks={actionLinks} />
<Table withHeader dataUrl="/rest/report-templates-table" columns={columns} actions={actions} />
</div>
);
}

14
client/src/users/List.js
View file

@ -11,7 +11,7 @@ export default class List extends Component {
render() {
const t = this.props.t;
let actionLinks;
let actions;
const columns = [
{ data: 0, title: "#" },
@ -21,10 +21,12 @@ export default class List extends Component {
if (mailtrainConfig.isAuthMethodLocal) {
columns.push({ data: 2, title: "Full Name" });
actionLinks = [{
label: 'Edit',
link: data => '/users/edit/' + data[0]
}];
actions = data => [
{
label: 'Edit',
link: '/users/edit/' + data[0]
}
];
}
return (
@ -35,7 +37,7 @@ export default class List extends Component {
<Title>{t('Users')}</Title>
<Table withHeader dataUrl="/rest/users-table" columns={columns} actionLinks={actionLinks} />
<Table withHeader dataUrl="/rest/users-table" columns={columns} actions={actions} />
</div>
);
}

262
lib/models/reports-REMOVE.js
View file

@ -1,262 +0,0 @@
'use strict';
const db = require('../db');
const tableHelpers = require('../table-helpers');
const fields = require('./fields');
const reportTemplates = require('./report-templates');
const tools = require('../tools');
const _ = require('../translate')._;
const allowedKeys = ['name', 'description', 'report_template', 'params'];
const ReportState = {
SCHEDULED: 0,
PROCESSING: 1,
FINISHED: 2,
FAILED: 3,
MAX: 4
};
module.exports.ReportState = ReportState;
module.exports.list = (start, limit, callback) => {
tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback);
};
module.exports.listWithState = (state, start, limit, callback) => {
tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback);
};
module.exports.filter = (request, callback) => {
tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback);
};
module.exports.get = (id, callback) => {
id = Number(id) || 0;
if (id < 1) {
return callback(new Error(_('Missing report ID')));
}
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
connection.query('SELECT * FROM reports WHERE id=?', [id], (err, rows) => {
connection.release();
if (err) {
return callback(err);
}
if (!rows || !rows.length) {
return callback(null, false);
}
const template = tools.convertKeys(rows[0]);
const params = template.params.trim();
if (params !== '') {
try {
template.paramsObject = JSON.parse(params);
} catch (err) {
return callback(err);
}
} else {
template.params = {};
}
return callback(null, template);
});
});
};
// This method is not supposed to be used for unsanitized inputs. It does not do any checks.
module.exports.updateFields = (id, fieldValueMap, callback) => {
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
const clauses = [];
const values = [];
for (let key of Object.keys(fieldValueMap)) {
clauses.push(tools.toDbKey(key) + '=?');
values.push(fieldValueMap[key]);
}
values.push(id);
const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1';
connection.query(query, values, (err, result) => {
connection.release();
if (err) {
return callback(err);
}
return callback(null, result && result.affectedRows || false);
});
});
};
module.exports.createOrUpdate = (createMode, report, callback) => {
report = report || {};
const id = 'id' in report ? Number(report.id) : 0;
if (!createMode && id < 1) {
return callback(new Error(_('Missing report ID')));
}
const name = (report.name || '').toString().trim();
if (!name) {
return callback(new Error(_('Report name must be set')));
}
const reportTemplateId = Number(report.reportTemplate);
reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
if (err) {
return callback(err);
}
const params = report.paramsObject;
for (const spec of reportTemplate.userFieldsObject) {
if (params[spec.id].length < spec.minOccurences) {
return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.')));
}
if (params[spec.id].length > spec.maxOccurences) {
return callback(new Error(_('At most ' + spec.minOccurences + ' rows in "' + spec.name + '" can be selected.')));
}
}
const keys = ['name', 'params'];
const values = [name, JSON.stringify(params)];
Object.keys(report).forEach(key => {
let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim();
key = tools.toDbKey(key);
if (key === 'description') {
value = tools.purifyHTML(value);
}
if (allowedKeys.indexOf(key) >= 0 && keys.indexOf(key) < 0) {
keys.push(key);
values.push(value);
}
});
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
let query;
if (createMode) {
query = 'INSERT INTO reports (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
} else {
query = 'UPDATE reports SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1';
values.push(id);
}
connection.query(query, values, (err, result) => {
connection.release();
if (err) {
return callback(err);
}
if (createMode) {
return callback(null, result && result.insertId || false);
} else {
return callback(null, result && result.affectedRows || false);
}
});
});
});
};
module.exports.delete = (id, callback) => {
id = Number(id) || 0;
if (id < 1) {
return callback(new Error(_('Missing report ID')));
}
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
connection.query('DELETE FROM reports WHERE id=? LIMIT 1', [id], (err, result) => {
connection.release();
if (err) {
return callback(err);
}
const affected = result && result.affectedRows || 0;
return callback(err, affected);
});
});
};
const campaignFieldsMapping = {
tracker_count: 'tracker.count',
country: 'tracker.country',
device_type: 'tracker.device_type',
status: 'campaign.status',
first_name: 'subscribers.first_name',
last_name: 'subscribers.last_name',
email: 'subscribers.email'
};
module.exports.getCampaignResults = (campaign, select, clause, callback) => {
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
fields.list(campaign.list, (err, fieldList) => {
if (err) {
return callback(err);
}
const fieldsMapping = fieldList.reduce((map, field) => {
map[customFieldName(field.key)] = 'subscribers.' + field.column;
return map;
}, Object.assign({}, campaignFieldsMapping));
let selFields = [];
for (let idx = 0; idx < select.length; idx++) {
const item = select[idx];
if (item in fieldsMapping) {
selFields.push(fieldsMapping[item] + ' AS ' + item);
} else if (item === '*') {
selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
} else {
selFields.push(item);
}
}
const query = 'SELECT ' + selFields.join(', ') + ' FROM `subscription__' + campaign.list + '` subscribers INNER JOIN `campaign__' + campaign.id + '` campaign on subscribers.id=campaign.subscription LEFT JOIN `campaign_tracker__' + campaign.id + '` tracker on subscribers.id=tracker.subscriber ' + clause;
connection.query(query, (err, results) => {
connection.release();
if (err) {
return callback(err);
}
return callback(null, results);
});
});
});
};
function customFieldName(id) {
return id.replace(/MERGE_/, 'CUSTOM_').toLowerCase();
}

9
lib/models/subscriptions.js
View file

@ -10,14 +10,7 @@ let segments = require('./segments');
let _ = require('../translate')._;
let tableHelpers = require('../table-helpers');
const Status = {
SUBSCRIBED: 1,
UNSUBSCRIBED: 2,
BOUNCED: 3,
COMPLAINED: 4,
MAX: 5
};
const Status = require('../../models/subscriptions').Status;
module.exports.Status = Status;
module.exports.list = (listId, start, limit, callback) => {

9
lib/report-processor.js
View file

@ -30,7 +30,7 @@ function startWorker(report) {
}
try {
await reports.update(report.id, fields);
await reports.updateFields(report.id, fields);
setImmediate(tryStartWorkers);
} catch (err) {
log.error('ReportProcessor', err);
@ -47,7 +47,7 @@ function startWorker(report) {
};
try {
await reports.update(report.id, fields);
await reports.updateFields(report.id, fields);
setImmediate(tryStartWorkers);
} catch (err) {
log.error('ReportProcessor', err);
@ -83,7 +83,7 @@ async function tryStartWorkers() {
log.info('ReportProcessor', 'Starting worker');
const report = reportList[0];
await report.updateFields(report.id, {state: reports.ReportState.PROCESSING});
await reports.updateFields(report.id, {state: reports.ReportState.PROCESSING});
startWorker(report);
} else {
@ -102,7 +102,7 @@ async function tryStartWorkers() {
module.exports.start = async reportId => {
if (!workers[reportId]) {
log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
await reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null});
await reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, last_run: null});
tryStartWorkers();
} else {
log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
@ -124,6 +124,7 @@ module.exports.stop = async reportId => {
module.exports.init = async () => {
try {
await reports.bulkChangeState(reports.ReportState.PROCESSING, reports.ReportState.SCHEDULED);
tryStartWorkers();
} catch (err) {
log.error('ReportProcessor', err);
}

12
models/campaigns.js
View file

@ -2,12 +2,22 @@
const knex = require('../lib/knex');
const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
async function listDTAjax(params) {
return await dtHelpers.ajaxList(params, tx => tx('campaigns'), ['campaigns.id', 'campaigns.name', 'campaigns.description', 'campaigns.status', 'campaigns.created']);
}
async function getById(id) {
const entity = await knex('campaigns').where('id', id).first();
if (!entity) {
throw new interoperableErrors.NotFoundError();
}
return entity;
}
module.exports = {
listDTAjax
listDTAjax,
getById
};

11
models/fields.js Normal file
View file

@ -0,0 +1,11 @@
'use strict';
const knex = require('../lib/knex');
const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const fieldsLegacy = require('../lib/models/fields');
const bluebird = require('bluebird');
module.exports = {
list: bluebird.promisify(fieldsLegacy.list)
};

13
models/lists.js
View file

@ -2,12 +2,23 @@
const knex = require('../lib/knex');
const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
async function listDTAjax(params) {
return await dtHelpers.ajaxList(params, tx => tx('lists'), ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description']);
}
async function getById(id) {
const entity = await knex('lists').where('id', id).first();
if (!entity) {
throw new interoperableErrors.NotFoundError();
}
return entity;
}
module.exports = {
listDTAjax
listDTAjax,
getById
};

91
models/reports.js
View file

@ -5,24 +5,19 @@ const hasher = require('node-object-hash')();
const { enforce, filterObject } = require('../lib/helpers');
const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const fields = require('./fields');
const ReportState = require('../shared/reports').ReportState;
const allowedKeys = new Set(['name', 'description', 'report_template', 'params']);
const ReportState = {
SCHEDULED: 0,
PROCESSING: 1,
FINISHED: 2,
FAILED: 3,
MAX: 4
};
function hash(entity) {
return hasher.hash(filterObject(entity, allowedKeys));
}
async function getByIdWithUserFields(id) {
const entity = await knex('reports').where('reports.id', id).innerJoin('report_templates', 'reports.report_template', 'report_templates.id').select(['reports.id', 'reports.name', 'reports.description', 'reports.report_template', 'reports.params', 'report_templates.user_fields']).first();
async function getByIdWithTemplate(id) {
const entity = await knex('reports').where('reports.id', id).innerJoin('report_templates', 'reports.report_template', 'report_templates.id').select(['reports.id', 'reports.name', 'reports.description', 'reports.report_template', 'reports.params', 'reports.state', 'report_templates.user_fields', 'report_templates.mime_type', 'report_templates.hbs', 'report_templates.js']).first();
if (!entity) {
throw new interoperableErrors.NotFoundError();
}
@ -34,10 +29,11 @@ async function getByIdWithUserFields(id) {
}
async function listDTAjax(params) {
return await dtHelpers.ajaxList(params, tx => tx('reports').innerJoin('report_templates', 'reports.report_template', 'report_templates.id'), ['reports.id', 'reports.name', 'report_templates.name', 'reports.description', 'reports.last_run', 'reports.state']);
return await dtHelpers.ajaxList(params, tx => tx('reports').innerJoin('report_templates', 'reports.report_template', 'report_templates.id'), ['reports.id', 'reports.name', 'report_templates.name', 'reports.description', 'reports.last_run', 'reports.state', 'report_templates.mime_type']);
}
async function create(entity) {
let id;
await knex.transaction(async tx => {
if (!await tx('report_templates').select(['id']).where('id', entity.report_template).first()) {
throw new interoperableErrors.DependencyNotFoundError();
@ -45,10 +41,12 @@ async function create(entity) {
entity.params = JSON.stringify(entity.params);
const id = await tx('reports').insert(filterObject(entity, allowedKeys));
return id;
id = await tx('reports').insert(filterObject(entity, allowedKeys));
});
const reportProcessor = require('../lib/report-processor');
await reportProcessor.start(id);
return id;
}
async function updateWithConsistencyCheck(entity) {
@ -71,8 +69,15 @@ async function updateWithConsistencyCheck(entity) {
entity.params = JSON.stringify(entity.params);
await tx('reports').where('id', entity.id).update(filterObject(entity, allowedKeys));
const filteredUpdates = filterObject(entity, allowedKeys);
filteredUpdates.state = ReportState.SCHEDULED;
await tx('reports').where('id', entity.id).update(filteredUpdates);
});
// This require is here to avoid cyclic dependency
const reportProcessor = require('../lib/report-processor');
await reportProcessor.start(entity.id);
}
async function remove(id) {
@ -92,16 +97,68 @@ async function bulkChangeState(oldState, newState) {
}
const campaignFieldsMapping = {
tracker_count: 'tracker.count',
country: 'tracker.country',
device_type: 'tracker.device_type',
status: 'campaign.status',
first_name: 'subscribers.first_name',
last_name: 'subscribers.last_name',
email: 'subscribers.email'
};
function customFieldName(id) {
return id.replace(/MERGE_/, 'CUSTOM_').toLowerCase();
}
async function getCampaignResults(campaign, select, extra) {
const fieldList = await fields.list(campaign.list);
const fieldsMapping = fieldList.reduce((map, field) => {
/* Dropdowns and checkboxes are aggregated. As such, they have field.column == null and the options are in field.options.
TODO - For the time being, we ignore groupped fields. */
if (field.column) {
map[customFieldName(field.key)] = 'subscribers.' + field.column;
}
return map;
}, Object.assign({}, campaignFieldsMapping));
let selFields = [];
for (let idx = 0; idx < select.length; idx++) {
const item = select[idx];
if (item in fieldsMapping) {
selFields.push(fieldsMapping[item] + ' AS ' + item);
} else if (item === '*') {
selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
} else {
selFields.push(item);
}
}
let query = knex(`subscription__${campaign.list} AS subscribers`)
.innerJoin(`campaign__${campaign.id} AS campaign`, 'subscribers.id', 'campaign.subscription')
.leftJoin(`campaign_tracker__${campaign.id} AS tracker`, 'subscribers.id', 'tracker.subscriber')
.select(selFields);
if (extra) {
query = extra(query);
}
return await query;
}
module.exports = {
ReportState,
hash,
getByIdWithUserFields,
getByIdWithTemplate,
listDTAjax,
create,
updateWithConsistencyCheck,
remove,
updateFields,
listByState,
bulkChangeState
bulkChangeState,
getCampaignResults
};

23
models/subscriptions.js Normal file
View file

@ -0,0 +1,23 @@
'use strict';
const knex = require('../lib/knex');
const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const Status = {
SUBSCRIBED: 1,
UNSUBSCRIBED: 2,
BOUNCED: 3,
COMPLAINED: 4,
MAX: 5
};
async function list(listId) {
return await knex(`subscription__${listId}`);
}
module.exports = {
Status,
list
};

406
routes/reports-REMOVE.js
View file

@ -1,406 +0,0 @@
'use strict';
const express = require('express');
const passport = require('../lib/passport');
const router = new express.Router();
const _ = require('../lib/translate')._;
const reportTemplates = require('../lib/models/report-templates');
const reports = require('../lib/models/reports-REMOVE');
const reportProcessor = require('../lib/report-processor');
const campaigns = require('../lib/models/campaigns');
const lists = require('../lib/models/lists');
const tools = require('../lib/tools');
const fileHelpers = require('../lib/file-helpers');
const util = require('util');
const htmlescape = require('escape-html');
const striptags = require('striptags');
const fs = require('fs');
const hbs = require('hbs');
router.all('/*', (req, res, next) => {
if (!req.user) {
req.flash('danger', _('Need to be logged in to access restricted content'));
return res.redirect('/account/login?next=' + encodeURIComponent(req.originalUrl));
}
res.setSelectedMenu('reports');
next();
});
router.get('/', (req, res) => {
res.render('reports/reports', {
title: _('Reports')
});
});
router.post('/ajax', (req, res) => {
reports.filter(req.body, (err, data, total, filteredTotal) => {
if (err) {
return res.json({
error: err.message || err,
data: []
});
}
res.json({
draw: req.body.draw,
recordsTotal: total,
recordsFiltered: filteredTotal,
data: data.map((row, i) => [
(Number(req.body.start) || 0) + 1 + i,
htmlescape(row.name || ''),
htmlescape(row.reportTemplateName || ''),
htmlescape(striptags(row.description) || ''),
getRowLastRun(row),
getRowActions(row)
])
});
});
});
router.get('/row/ajax/:id', (req, res) => {
respondRowActions(req.params.id, res);
});
router.get('/start/ajax/:id', (req, res) => {
reportProcessor.start(req.params.id, () => {
respondRowActions(req.params.id, res);
});
});
router.get('/stop/ajax/:id', (req, res) => {
reportProcessor.stop(req.params.id, () => {
respondRowActions(req.params.id, res);
});
});
router.get('/create', passport.csrfProtection, (req, res) => {
const reqData = req.query;
reqData.csrfToken = req.csrfToken();
reqData.title = _('Create Report');
reqData.useEditor = true;
reportTemplates.quicklist((err, items) => {
if (err) {
req.flash('danger', err.message || err);
return res.redirect('/reports');
}
const reportTemplateId = Number(reqData.reportTemplate);
if (reportTemplateId) {
items.forEach(item => {
if (item.id === reportTemplateId) {
item.selected = true;
}
});
}
reqData.reportTemplates = items;
if (!reportTemplateId) {
res.render('reports/create-select-template', reqData);
} else {
addUserFields(reportTemplateId, reqData, null, (err, data) => {
if (err) {
req.flash('danger', err.message || err);
return res.redirect('/reports');
}
res.render('reports/create', data);
});
}
});
});
router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
const reqData = req.body;
const reportTemplateId = Number(reqData.reportTemplate);
addParamsObject(reportTemplateId, reqData, (err, data) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not create report'));
return res.redirect('/reports/create?' + tools.queryParams(data));
}
reports.createOrUpdate(true, data, (err, id) => {
if (err || !id) {
req.flash('danger', err && err.message || err || _('Could not create report'));
return res.redirect('/reports/create?' + tools.queryParams(data));
}
reportProcessor.start(id, () => {
req.flash('success', util.format(_('Report “%s” created'), data.name));
res.redirect('/reports');
});
});
});
});
router.get('/edit/:id', passport.csrfProtection, (req, res) => {
const reqData = req.query;
reports.get(req.params.id, (err, report) => {
if (err || !report) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
report.csrfToken = req.csrfToken();
report.title = _('Edit Report');
report.useEditor = true;
reportTemplates.quicklist((err, items) => {
if (err) {
req.flash('danger', err.message || err);
return res.redirect('/');
}
const reportTemplateId = report.reportTemplate;
items.forEach(item => {
if (item.id === reportTemplateId) {
item.selected = true;
}
});
report.reportTemplates = items;
addUserFields(reportTemplateId, reqData, report, (err, data) => {
if (err) {
req.flash('danger', err.message || err);
return res.redirect('/reports');
}
res.render('reports/edit', data);
});
});
});
});
router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
const reqData = req.body;
const reportTemplateId = Number(reqData.reportTemplate);
addParamsObject(reportTemplateId, reqData, (err, data) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not update report'));
return res.redirect('/reports/create?' + tools.queryParams(data));
}
reports.createOrUpdate(false, data, (err, updated) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not update report'));
return res.redirect('/reports/edit/' + data.id + '?' + tools.queryParams(data));
} else if (updated) {
req.flash('success', _('Report updated'));
} else {
req.flash('info', _('Report not updated'));
}
return res.redirect('/reports');
});
});
});
router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
reports.delete(req.body.id, (err, deleted) => {
if (err) {
req.flash('danger', err && err.message || err);
} else if (deleted) {
req.flash('success', _('Report deleted'));
} else {
req.flash('info', _('Could not delete specified report'));
}
return res.redirect('/reports');
});
});
router.get('/view/:id', (req, res) => {
reports.get(req.params.id, (err, report) => {
if (err || !report) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not find report template'));
return res.redirect('/reports');
}
if (report.state == reports.ReportState.FINISHED) {
if (reportTemplate.mimeType == 'text/html') {
fs.readFile(fileHelpers.getReportContentFile(report), (err, reportContent) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
const data = {
report: new hbs.handlebars.SafeString(reportContent),
title: report.name
};
res.render('reports/view', data);
});
} else if (reportTemplate.mimeType == 'text/csv') {
const headers = {
'Content-Disposition': 'attachment;filename=' + fileHelpers.nameToFileName(report.name) + '.csv',
'Content-Type': 'text/csv'
};
res.sendFile(fileHelpers.getReportContentFile(report), {headers: headers});
} else {
req.flash('danger', _('Unknown type of template'));
res.redirect('/reports');
}
} else {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
});
});
});
router.get('/output/:id', (req, res) => {
reports.get(req.params.id, (err, report) => {
if (err || !report) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
fs.readFile(fileHelpers.getReportOutputFile(report), (err, output) => {
let data = {
title: 'Output for report ' + report.name
};
if (err) {
data.error = 'No output.';
} else {
data.output = output;
}
res.render('reports/output', data);
});
});
});
function getRowLastRun(row) {
return '<span id="row-last-run-' + row.id + '">' + (row.lastRun ? '<span class="datestring" data-date="' + row.lastRun.toISOString() + '" title="' + row.lastRun.toISOString() + '">' + row.lastRun.toISOString() + '</span>' : '') + '</span>';
}
function getRowActions(row) {
/* FIXME: add csrf protection to stop and refresh actions */
let requestRefresh = false;
let view, startStop;
let topic = 'data-topic-id="' + row.id + '"';
if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) {
view = '<span class="row-action glyphicon glyphicon-hourglass" aria-hidden="true" title="Processing"></span>';
startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/stop" ' + topic + ' title="Stop"><span class="glyphicon glyphicon-stop" aria-hidden="true"></span></a>';
requestRefresh = true;
} else if (row.state == reports.ReportState.FINISHED) {
let icon = 'eye-open';
if (row.mimeType == 'text/csv') icon = 'download-alt';
view = '<a class="row-action" href="/reports/view/' + row.id + '" title="View report"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a>';
startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
} else if (row.state == reports.ReportState.FAILED) {
view = '<span class="row-action glyphicon glyphicon-thumbs-down" aria-hidden="true" title="Report generation failed"></span>';
startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
}
let actions = view;
actions += '<a class="row-action" href="/reports/output/' + row.id + '" title="View console output"><span class="glyphicon glyphicon-modal-window" aria-hidden="true"></span></a>';
actions += startStop;
actions += '<a class="row-action" href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true" title="Edit"></span></a>';
return '<span id="row-actions-' + row.id + '"' + (requestRefresh ? ' class="row-actions ajax-refresh" data-interval="5" data-topic-url="/reports/row" ' + topic : ' class="row-actions"') + '>' +
actions +
'</span>';
}
function respondRowActions(id, res) {
reports.get(id, (err, report) => {
if (err) {
return res.json({
error: err,
});
}
const data = {};
data['#row-last-run-' + id] = getRowLastRun(report);
data['#row-actions-' + id] = getRowActions(report);
res.json(data);
});
}
function addUserFields(reportTemplateId, reqData, report, callback) {
reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
if (err) {
return callback(err);
}
const userFields = [];
for (const spec of reportTemplate.userFieldsObject) {
let value = '';
if ((spec.id + 'Selection') in reqData) {
value = reqData[spec.id + 'Selection'];
} else if (report && report.paramsObject && spec.id in report.paramsObject) {
value = report.paramsObject[spec.id].join(',');
}
userFields.push({
'id': spec.id,
'name': spec.name,
'type': spec.type,
'value': value,
'isMulti': !(spec.minOccurences == 1 && spec.maxOccurences == 1)
});
}
const data = report ? report : reqData;
data.userFields = userFields;
callback(null, data);
});
}
function addParamsObject(reportTemplateId, data, callback) {
reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
if (err) {
return callback(err);
}
const paramsObject = {};
for (const spec of reportTemplate.userFieldsObject) {
const sel = data[spec.id + 'Selection'];
if (!sel) {
paramsObject[spec.id] = [];
} else {
paramsObject[spec.id] = sel.split(',').map(item => Number(item));
}
}
data.paramsObject = paramsObject;
callback(null, data);
});
}
module.exports = router;

26
routes/reports.js Normal file
View file

@ -0,0 +1,26 @@
'use strict';
const passport = require('../lib/passport');
const _ = require('../lib/translate')._;
const reports = require('../models/reports');
const fileHelpers = require('../lib/file-helpers');
const router = require('../lib/router-async').create();
router.getAsync('/download/:id', passport.loggedIn, async (req, res) => {
const report = await reports.getByIdWithTemplate(req.params.id);
if (report.state == reports.ReportState.FINISHED) {
const headers = {
'Content-Disposition': 'attachment;filename=' + fileHelpers.nameToFileName(report.name) + '.csv',
'Content-Type': report.mime_type
};
res.sendFile(fileHelpers.getReportContentFile(report), {headers: headers});
} else {
return res.status(404).send(_('Report not found'));
}
});
module.exports = router;

18
routes/rest/reports.js
View file

@ -4,12 +4,13 @@ const passport = require('../../lib/passport');
const _ = require('../../lib/translate')._;
const reports = require('../../models/reports');
const reportProcessor = require('../../lib/report-processor');
const fileHelpers = require('../../lib/file-helpers');
const router = require('../../lib/router-async').create();
router.getAsync('/reports/:reportId', passport.loggedIn, async (req, res) => {
const report = await reports.getByIdWithUserFields(req.params.reportId);
const report = await reports.getByIdWithTemplate(req.params.reportId);
report.hash = reports.hash(report);
return res.json(report);
});
@ -38,14 +39,23 @@ router.postAsync('/reports-table', passport.loggedIn, async (req, res) => {
router.postAsync('/report-start/:id', passport.loggedIn, passport.csrfProtection, async (req, res) => {
await reportProcessor.start(req.params.id);
// TODO
res.json();
});
router.postAsync('/report-stop/:id', async (req, res) => {
await reportProcessor.stop(req.params.id);
// TODO
res.json();
});
router.getAsync('/report-content/:id', async (req, res) => {
const report = await reports.getByIdWithTemplate(req.params.id);
res.sendFile(fileHelpers.getReportContentFile(report));
});
router.getAsync('/report-output/:id', async (req, res) => {
const report = await reports.getByIdWithTemplate(req.params.id);
res.sendFile(fileHelpers.getReportOutputFile(report));
});
module.exports = router;

1
services/executor.js
View file

@ -129,3 +129,4 @@ process.on('message', msg => {
process.send({
type: 'executor-started'
});

13
shared/reports.js Normal file
View file

@ -0,0 +1,13 @@
'use strict';
const ReportState = {
SCHEDULED: 0,
PROCESSING: 1,
FINISHED: 2,
FAILED: 3,
MAX: 4
};
module.exports = {
ReportState
};

196
workers/reports/report-processor.js
View file

@ -1,10 +1,10 @@
'use strict';
const reports = require('../../lib/models/reports-REMOVE');
const reportTemplates = require('../../lib/models/report-templates');
const lists = require('../../lib/models/lists');
const subscriptions = require('../../lib/models/subscriptions');
const campaigns = require('../../lib/models/campaigns');
const reports = require('../../models/reports');
const reportTemplates = require('../../models/report-templates');
const lists = require('../../models/lists');
const subscriptions = require('../../models/subscriptions');
const campaigns = require('../../models/campaigns');
const handlebars = require('handlebars');
const handlebarsHelpers = require('../../lib/handlebars-helpers');
const _ = require('../../lib/translate')._;
@ -12,136 +12,82 @@ const hbs = require('hbs');
const vm = require('vm');
const log = require('npmlog');
const fs = require('fs');
const knex = require('../../lib/knex');
handlebarsHelpers.registerHelpers(handlebars);
let reportId = Number(process.argv[2]);
let reportDir;
function resolveEntities(getter, ids, callback) {
const idsRemaining = ids.slice();
const resolved = [];
function doWork() {
if (idsRemaining.length == 0) {
return callback(null, resolved);
}
getter(idsRemaining.shift(), (err, entity) => {
if (err) {
return callback(err);
}
resolved.push(entity);
return doWork();
});
}
setImmediate(doWork);
}
const userFieldTypeToGetter = {
'campaign': (id, callback) => campaigns.get(id, false, callback),
'list': lists.get
const userFieldGetters = {
'campaign': campaigns.getById,
'list': lists.getById
};
function resolveUserFields(userFields, params, callback) {
const userFieldsRemaining = userFields.slice();
const resolved = {};
function doWork() {
if (userFieldsRemaining.length == 0) {
return callback(null, resolved);
}
async function main() {
try {
const reportId = Number(process.argv[2]);
const spec = userFieldsRemaining.shift();
const getter = userFieldTypeToGetter[spec.type];
const report = await reports.getByIdWithTemplate(reportId);
if (getter) {
return resolveEntities(getter, params[spec.id], (err, entities) => {
if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
resolved[spec.id] = entities[0];
} else {
resolved[spec.id] = entities;
}
const inputs = {};
doWork();
});
} else {
return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
}
}
setImmediate(doWork);
}
function doneSuccess() {
process.exit(0);
}
function doneFail() {
process.exit(1)
}
reports.get(reportId, (err, report) => {
if (err || !report) {
log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
doneFail();
}
reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
if (err) {
log.error('reports', err && err.message || err || _('Could not find report template'));
doneFail();
}
resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
if (err) {
log.error('reports', err.message || err);
doneFail();
for (const spec of report.user_fields) {
const getter = userFieldGetters[spec.type];
if (!getter) {
throw new Error(_('Unknown user field type "' + spec.type + '".'));
}
const campaignsProxy = {
results: reports.getCampaignResults,
list: campaigns.list,
get: campaigns.get
};
const subscriptionsProxy = {
list: subscriptions.list
};
const sandbox = {
console,
campaigns: campaignsProxy,
subscriptions: subscriptionsProxy,
inputs,
callback: (err, outputs) => {
if (err) {
log.error('reports', err.message || err);
doneFail();
}
const hbsTmpl = handlebars.compile(reportTemplate.hbs);
const reportText = hbsTmpl(outputs);
process.stdout.write(reportText);
doneSuccess();
}
};
const script = new vm.Script(reportTemplate.js);
try {
script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000});
} catch (err) {
console.error(err);
doneFail();
const entities = [];
for (const id of report.params[spec.id]) {
entities.push(await getter(id));
}
});
});
});
if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
inputs[spec.id] = entities[0];
} else {
inputs[spec.id] = entities;
}
}
const campaignsProxy = {
getResults: reports.getCampaignResults,
getById: campaigns.getById
};
const subscriptionsProxy = {
list: subscriptions.list
};
const sandbox = {
console,
campaigns: campaignsProxy,
subscriptions: subscriptionsProxy,
knex,
process,
inputs,
render: data => {
const hbsTmpl = handlebars.compile(report.hbs);
const reportText = hbsTmpl(data);
process.stdout.write(reportText);
}
};
const js =
'(async function() {' +
report.js +
'})().then(() => process.exit(0)).catch(err => { console.error(err); process.exit(1); })';
const script = new vm.Script(js);
script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000});
} catch (err) {
console.error(err);
process.exit(1);
}
}
main();