From d63eed9ca9671680d031d5aa4aa590fc0ed936f6 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Thu, 13 Jul 2017 13:27:03 +0200 Subject: [PATCH] Reports ported to ReactJS and Knex Note that the interface for the custom JS code inside a report template has changed. It now offers promise-based interface and exposes knex. --- app.js | 41 ++- client/src/lib/form.js | 64 +++-- client/src/lib/page.css | 4 +- client/src/lib/table.js | 87 +++++- client/src/lib/tree.js | 21 +- client/src/namespaces/List.js | 6 +- client/src/reports/List.js | 88 +++++- client/src/reports/Output.js | 53 ++++ client/src/reports/View.js | 57 ++++ client/src/reports/root.js | 12 + client/src/reports/templates/CUD.js | 53 ++-- client/src/reports/templates/List.js | 12 +- client/src/users/List.js | 14 +- lib/models/reports-REMOVE.js | 262 ----------------- lib/models/subscriptions.js | 9 +- lib/report-processor.js | 9 +- models/campaigns.js | 12 +- models/fields.js | 11 + models/lists.js | 13 +- models/reports.js | 91 ++++-- models/subscriptions.js | 23 ++ routes/reports-REMOVE.js | 406 --------------------------- routes/reports.js | 26 ++ routes/rest/reports.js | 18 +- services/executor.js | 1 + shared/reports.js | 13 + workers/reports/report-processor.js | 196 +++++-------- 27 files changed, 649 insertions(+), 953 deletions(-) create mode 100644 client/src/reports/Output.js create mode 100644 client/src/reports/View.js delete mode 100644 lib/models/reports-REMOVE.js create mode 100644 models/fields.js create mode 100644 models/subscriptions.js delete mode 100644 routes/reports-REMOVE.js create mode 100644 routes/reports.js create mode 100644 shared/reports.js diff --git a/app.js b/app.js index 14912aa6..7f0f9023 100644 --- a/app.js +++ b/app.js @@ -40,6 +40,9 @@ const editorapi = require('./routes/editorapi'); const grapejs = require('./routes/grapejs'); const mosaico = require('./routes/mosaico'); +// These are routes for the new React-based client +const reports = require('./routes/reports'); + const namespacesRest = require('./routes/rest/namespaces'); const usersRest = require('./routes/rest/users'); const accountRest = require('./routes/rest/account'); @@ -242,18 +245,22 @@ app.use('/editorapi', editorapi); app.use('/grapejs', grapejs); app.use('/mosaico', mosaico); + +if (config.reports && config.reports.enabled === true) { + app.use('/reports', reports); +} + /* FIXME - this should be removed once we bind the ReactJS client to / */ app.use('/users', usersLegacyIntegration); app.use('/namespaces', namespacesLegacyIntegration); app.use('/account', accountLegacyIntegration); if (config.reports && config.reports.enabled === true) { + app.use('/reports', reports); app.use('/reports', reportsLegacyIntegration); } - /* ------------------------------------------------------------------- */ - app.all('/rest/*', (req, res, next) => { req.needsJSONResponse = true; next(); @@ -301,11 +308,16 @@ if (app.get('env') === 'development') { res.status(err.status || 500).json(resp); } else { - res.status(err.status || 500); - res.render('error', { - message: err.message, - error: err - }); + if (err instanceof interoperableErrors.NotLoggedInError) { + req.flash('danger', _('Need to be logged in to access restricted content')); + return res.redirect('/account/login?next=' + encodeURIComponent(req.originalUrl)); + } else { + res.status(err.status || 500); + res.render('error', { + message: err.message, + error: err + }); + } } }); @@ -331,11 +343,16 @@ if (app.get('env') === 'development') { res.status(err.status || 500).json(resp); } else { - res.status(err.status || 500); - res.render('error', { - message: err.message, - error: {} - }); + if (err instanceof interoperableErrors.NotLoggedInError) { + req.flash('danger', _('Need to be logged in to access restricted content')); + return res.redirect('/account/login?next=' + encodeURIComponent(req.originalUrl)); + } else { + res.status(err.status || 500); + res.render('error', { + message: err.message, + error: {} + }); + } } }); } diff --git a/client/src/lib/form.js b/client/src/lib/form.js index 2453ec32..468076f2 100644 --- a/client/src/lib/form.js +++ b/client/src/lib/form.js @@ -49,36 +49,6 @@ class Form extends Component { }; } - static async handleChangedError(owner, fn) { - try { - await fn(); - } catch (error) { - if (error instanceof interoperableErrors.ChangedError) { - owner.disableForm(); - owner.setFormStatusMessage('danger', - - {t('Your updates cannot be saved.')}{' '} - {t('Someone else has introduced modification in the meantime. Refresh your page to start anew with fresh data. Please note that your changes will be lost.')} - - ); - return; - } - - if (error instanceof interoperableErrors.NotFoundError) { - owner.disableForm(); - owner.setFormStatusMessage('danger', - - {t('Your updates cannot be saved.')}{' '} - {t('It seems that someone else has deleted the entity in the meantime.')} - - ); - return; - } - - throw error; - } - } - @withAsyncErrorHandler async onSubmit(evt) { const t = this.props.t; @@ -88,7 +58,7 @@ class Form extends Component { evt.preventDefault(); if (this.props.onSubmitAsync) { - await Form.handleChangedError(owner, async () => await this.props.onSubmitAsync(evt)); + await this.formHandleChangedError(async () => await this.props.onSubmitAsync(evt)); } } @@ -520,6 +490,7 @@ class ACEEditor extends Component { showPrintMargin={false} value={owner.getFormValue(id)} tabSize={2} + setOptions={{useWorker: false}} // This disables syntax check because it does not always work well (e.g. in case of JS code in report templates) /> ); } @@ -839,6 +810,37 @@ function withForm(target) { return this.state.formState.get('isDisabled'); }; + inst.formHandleChangedError = async function(fn) { + const t = this.props.t; + try { + await fn(); + } catch (error) { + if (error instanceof interoperableErrors.ChangedError) { + this.disableForm(); + this.setFormStatusMessage('danger', + + {t('Your updates cannot be saved.')}{' '} + {t('Someone else has introduced modification in the meantime. Refresh your page to start anew with fresh data. Please note that your changes will be lost.')} + + ); + return; + } + + if (error instanceof interoperableErrors.NotFoundError) { + this.disableForm(); + this.setFormStatusMessage('danger', + + {t('Your updates cannot be saved.')}{' '} + {t('It seems that someone else has deleted the entity in the meantime.')} + + ); + return; + } + + throw error; + } + }; + return target; } diff --git a/client/src/lib/page.css b/client/src/lib/page.css index fa84852d..029b6b67 100644 --- a/client/src/lib/page.css +++ b/client/src/lib/page.css @@ -11,11 +11,11 @@ padding-bottom: 5px; } -.mt-action-links > a { +.mt-action-links > * { margin-right: 8px; } -.mt-action-links > a:last-child { +.mt-action-links > *:last-child { margin-right: 0px; } diff --git a/client/src/lib/table.js b/client/src/lib/table.js index 670015c4..8a1b63a2 100644 --- a/client/src/lib/table.js +++ b/client/src/lib/table.js @@ -29,7 +29,7 @@ const TableSelectMode = { }; -@translate() +@translate(null, { withRef: true }) @withPageHelpers @withErrorHandling class Table extends Component { @@ -48,8 +48,9 @@ class Table extends Component { selectionAsArray: PropTypes.bool, onSelectionChangedAsync: PropTypes.func, onSelectionDataAsync: PropTypes.func, - actionLinks: PropTypes.array, - withHeader: PropTypes.bool + actions: PropTypes.func, + withHeader: PropTypes.bool, + refreshInterval: PropTypes.number } static defaultProps = { @@ -57,6 +58,12 @@ class Table extends Component { selectionKeyIndex: 0 } + refresh() { + if (this.table) { + this.table.rows().draw('page'); + } + } + getSelectionMap(props) { let selArray = []; if (props.selectMode === TableSelectMode.SINGLE && !this.props.selectionAsArray) { @@ -162,17 +169,56 @@ class Table extends Component { componentDidMount() { const columns = this.props.columns.slice(); - if (this.props.actionLinks) { - const actionLinks = this.props.actionLinks; - + if (this.props.actions) { const createdCellFn = (td, data) => { const linksContainer = jQuery(''); - for (const {label, link} of actionLinks) { - const dest = link(data); - const lnkHtml = ReactDOMServer.renderToStaticMarkup({label}); - const lnk = jQuery(lnkHtml); - lnk.click((evt) => { evt.preventDefault(); this.navigateTo(dest) }); - linksContainer.append(lnk); + + let actions = this.props.actions(data); + let options = {}; + + if (!Array.isArray(actions)) { + options = actions; + actions = actions.actions; + } + + for (const action of actions) { + if (action.action) { + const html = ReactDOMServer.renderToStaticMarkup({action.label}); + const elem = jQuery(html); + elem.click((evt) => { evt.preventDefault(); action.action(this) }); + linksContainer.append(elem); + + } else if (action.link) { + const html = ReactDOMServer.renderToStaticMarkup({action.label}); + const elem = jQuery(html); + elem.click((evt) => { evt.preventDefault(); this.navigateTo(action.link) }); + linksContainer.append(elem); + + } else if (action.href) { + const html = ReactDOMServer.renderToStaticMarkup({action.label}); + const elem = jQuery(html); + linksContainer.append(elem); + + } else { + const html = ReactDOMServer.renderToStaticMarkup(action.label); + const elem = jQuery(html); + linksContainer.append(elem); + } + } + + if (options.refreshTimeout) { + const currentMS = Date.now(); + + if (!this.refreshTimeoutAt || this.refreshTimeoutAt > currentMS + options.refreshTimeout) { + clearTimeout(this.refreshTimeoutId); + + this.refreshTimeoutAt = currentMS + options.refreshTimeout; + + this.refreshTimeoutId = setTimeout(() => { + this.refreshTimeoutAt = 0; + this.refresh(); + }, options.refreshTimeout); + } } jQuery(td).html(linksContainer); @@ -238,6 +284,15 @@ class Table extends Component { this.table = jQuery(this.domTable).DataTable(dtOptions); + if (this.props.refreshInterval) { + this.refreshIntervalId = setInterval(() => this.refresh(), this.props.refreshInterval); + } + + this.table.on('destroy.dt', () => { + clearInterval(this.refreshIntervalId); + clearTimeout(this.refreshTimeoutId); + }); + this.fetchSelectionData(); } @@ -306,6 +361,14 @@ class Table extends Component { } } +/* + Refreshes the table. This method is provided to allow programmatic refresh from a handler outside the table. + The reference to the table can be obtained by ref. + */ +Table.prototype.refresh = function() { + this.getWrappedInstance().refresh() +}; + export { Table, TableSelectMode diff --git a/client/src/lib/tree.js b/client/src/lib/tree.js index 27e66dd1..edfbe497 100644 --- a/client/src/lib/tree.js +++ b/client/src/lib/tree.js @@ -65,7 +65,7 @@ class TreeTable extends Component { selectMode: PropTypes.number, selection: PropTypes.oneOfType([PropTypes.array, PropTypes.string, PropTypes.number]), onSelectionChangedAsync: PropTypes.func, - actionLinks: PropTypes.array, + actions: PropTypes.func, withHeader: PropTypes.bool } @@ -100,19 +100,18 @@ class TreeTable extends Component { }; let createNodeFn; - if (this.props.actionLinks) { - const actionLinks = this.props.actionLinks; - + if (this.props.actions) { createNodeFn = (event, data) => { const node = data.node; const tdList = jQuery(node.tr).find(">td"); const linksContainer = jQuery(''); - for (const {label, link} of actionLinks) { - const dest = link(node.key); - const lnkHtml = ReactDOMServer.renderToStaticMarkup({label}); + + const actions = this.props.actions(node.key); + for (const {label, link} of actions) { + const lnkHtml = ReactDOMServer.renderToStaticMarkup({label}); const lnk = jQuery(lnkHtml); - lnk.click((evt) => { evt.preventDefault(); this.navigateTo(dest) }); + lnk.click((evt) => { evt.preventDefault(); this.navigateTo(link) }); linksContainer.append(lnk); } @@ -202,7 +201,7 @@ class TreeTable extends Component { render() { const t = this.props.t; const props = this.props; - const actionLinks = props.actionLinks; + const actions = props.actions; const withHeader = props.withHeader; let containerClass = 'mt-treetable-container'; @@ -223,14 +222,14 @@ class TreeTable extends Component { {t('Name')} - {actionLinks && } + {actions && } } - {actionLinks && } + {actions && } diff --git a/client/src/namespaces/List.js b/client/src/namespaces/List.js index a16ab205..af107c27 100644 --- a/client/src/namespaces/List.js +++ b/client/src/namespaces/List.js @@ -10,10 +10,10 @@ export default class List extends Component { render() { const t = this.props.t; - const actionLinks = [ + const actions = key => [ { label: 'Edit', - link: key => '/namespaces/edit/' + key + link: '/namespaces/edit/' + key } ]; @@ -25,7 +25,7 @@ export default class List extends Component { {t('Namespaces')} - + ); } diff --git a/client/src/reports/List.js b/client/src/reports/List.js index 08e271c5..1dc1de8f 100644 --- a/client/src/reports/List.js +++ b/client/src/reports/List.js @@ -4,24 +4,100 @@ import React, { Component } from 'react'; import { translate } from 'react-i18next'; import { Title, Toolbar, NavButton } from '../lib/page'; import { Table } from '../lib/table'; +import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling'; import moment from 'moment'; +import axios from '../lib/axios'; +import { ReportState } from '../../../shared/reports'; @translate() +@withErrorHandling export default class List extends Component { + + @withAsyncErrorHandler + async stop(table, id) { + await axios.post(`/rest/report-stop/${id}`); + table.refresh(); + } + + @withAsyncErrorHandler + async start(table, id) { + await axios.post(`/rest/report-start/${id}`); + table.refresh(); + } + render() { const t = this.props.t; - const actionLinks = [{ - label: 'Edit', - link: data => '/reports/edit/' + data[0] - }]; + const actions = data => { + let view, startStop, refreshTimeout; + + const state = data[5]; + const id = data[0]; + const mimeType = data[6]; + + if (state === ReportState.PROCESSING || state === ReportState.SCHEDULED) { + view = { + label: , + }; + + startStop = { + label: , + action: (table) => this.stop(table, id) + }; + + refreshTimeout = 1000; + } else if (state === ReportState.FINISHED) { + if (mimeType === 'text/html') { + view = { + label: , + link: `reports/view/${id}` + }; + } else if (mimeType === 'text/csv') { + view = { + label: , + href: `reports/download/${id}` + }; + } + + startStop = { + label: , + action: (table) => this.start(table, id) + }; + + } else if (state === ReportState.FAILED) { + view = { + label: , + }; + + startStop = { + label: , + action: (table) => this.start(table, id) + }; + } + + return { + refreshTimeout, + actions: [ + view, + { + label: , + link: `reports/output/${id}` + }, + startStop, + { + label: , + link: `/reports/edit/${id}` + } + ] + }; + }; const columns = [ { data: 0, title: "#" }, { data: 1, title: t('Name') }, { data: 2, title: t('Template') }, { data: 3, title: t('Description') }, - { data: 4, title: t('Last Run'), render: data => data ? moment(data).fromNow() : t('Not run yet') } + { data: 4, title: t('Created'), render: data => data ? moment(data).fromNow() : '' } ]; return ( @@ -33,7 +109,7 @@ export default class List extends Component { {t('Reports')} - +
); } diff --git a/client/src/reports/Output.js b/client/src/reports/Output.js new file mode 100644 index 00000000..b3df049b --- /dev/null +++ b/client/src/reports/Output.js @@ -0,0 +1,53 @@ +'use strict'; + +import React, { Component } from 'react'; +import { translate } from 'react-i18next'; +import { withPageHelpers, Title } from '../lib/page' +import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling'; +import axios from '../lib/axios'; + +@translate() +@withPageHelpers +@withErrorHandling +export default class Output extends Component { + constructor(props) { + super(props); + + this.state = { + output: null + }; + } + + @withAsyncErrorHandler + async loadOutput() { + const id = parseInt(this.props.match.params.id); + const outputResp = await axios.get(`/rest/report-output/${id}`); + const reportResp = await axios.get(`/rest/reports/${id}`); + + this.setState({ + output: outputResp.data, + report: reportResp.data, + }); + } + + componentDidMount() { + this.loadOutput(); + } + + render() { + const t = this.props.t; + + if (this.state.report) { + return ( +
+ {t('Output for report {{name}}', { name: this.state.report.name })} + + 
{this.state.output}
+
+ ); + } else { + return
{t('Loading report output ...')}
; + } + + } +} diff --git a/client/src/reports/View.js b/client/src/reports/View.js new file mode 100644 index 00000000..902ec073 --- /dev/null +++ b/client/src/reports/View.js @@ -0,0 +1,57 @@ +'use strict'; + +import React, { Component } from 'react'; +import { translate } from 'react-i18next'; +import { withPageHelpers, Title } from '../lib/page' +import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling'; +import axios from '../lib/axios'; +import { ReportState } from '../../../shared/reports'; + +@translate() +@withPageHelpers +@withErrorHandling +export default class View extends Component { + constructor(props) { + super(props); + + this.state = { + content: null + }; + } + + @withAsyncErrorHandler + async loadContent() { + const id = parseInt(this.props.match.params.id); + const contentResp = await axios.get(`/rest/report-content/${id}`); + const reportResp = await axios.get(`/rest/reports/${id}`); + + this.setState({ + content: contentResp.data, + report: reportResp.data + }); + } + + componentDidMount() { + this.loadContent(); + } + + render() { + const t = this.props.t; + + if (this.state.report) { + if (this.state.report.state === ReportState.FINISHED) { + return ( +
+ {t('Report {{name}}', { name: this.state.report.name })} + +
+
+ ); + } else { + return
{t('Report not generated')}
; + } + } else { + return
{t('Loading report ...')}
; + } + } +} diff --git a/client/src/reports/root.js b/client/src/reports/root.js index 416b4c80..49f9145a 100644 --- a/client/src/reports/root.js +++ b/client/src/reports/root.js @@ -8,6 +8,8 @@ import i18n from '../lib/i18n'; import { Section } from '../lib/page' import ReportsCUD from './CUD' import ReportsList from './List' +import ReportsView from './View' +import ReportsOutput from './Output' import ReportTemplatesCUD from './templates/CUD' import ReportTemplatesList from './templates/List' @@ -33,6 +35,16 @@ const getStructure = t => { title: t('Create Report'), render: props => () }, + view: { + title: t('View Report'), + params: [':id' ], + render: props => () + }, + output: { + title: t('View Report Output'), + params: [':id' ], + render: props => () + }, 'templates': { title: t('Templates'), link: '/reports/templates', diff --git a/client/src/reports/templates/CUD.js b/client/src/reports/templates/CUD.js index 03add471..4934fc9a 100644 --- a/client/src/reports/templates/CUD.js +++ b/client/src/reports/templates/CUD.js @@ -57,17 +57,8 @@ export default class CUD extends Component { ' }\n' + ']', js: - 'campaigns.results(inputs.campaign, ["*"], "", (err, results) => {\n' + - ' if (err) {\n' + - ' return callback(err);\n' + - ' }\n' + - '\n' + - ' const data = {\n' + - ' results: results\n' + - ' };\n' + - '\n' + - ' return callback(null, data);\n' + - '});', + 'const results = await campaigns.getResults(inputs.campaign, ["*"]);\n' + + 'render({ results });', hbs: '

{{title}}

\n' + '\n' + @@ -115,21 +106,17 @@ export default class CUD extends Component { ' }\n' + ']', js: - 'campaigns.results(inputs.campaign, ["custom_country", "count(*) AS count_all", "SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"], "GROUP BY custom_country", (err, results) => {\n' + - ' if (err) {\n' + - ' return callback(err);\n' + - ' }\n' + + 'const results = await campaigns.getResults(inputs.campaign, ["custom_country"], query =>\n' + + ' query.count("* AS count_all")\n' + + ' .select(knex.raw("SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"))\n' + + ' .groupBy("custom_country")\n' + + ');\n' + '\n' + - ' for (let row of results) {\n' + - ' row["percentage"] = Math.round((row.count_opened / row.count_all) * 100);\n' + - ' }\n' + + 'for (const row of results) {\n' + + ' row.percentage = Math.round((row.count_opened / row.count_all) * 100);\n' + + '}\n' + '\n' + - ' let data = {\n' + - ' results: results\n' + - ' };\n' + - '\n' + - ' return callback(null, data);\n' + - '});', + 'render({ results });', hbs: '

{{title}}

\n' + '\n' + @@ -189,17 +176,8 @@ export default class CUD extends Component { ' }\n' + ']', js: - 'subscriptions.list(inputs.list.id,0,0, (err, results) => {\n' + - ' if (err) {\n' + - ' return callback(err);\n' + - ' }\n' + - '\n' + - ' let data = {\n' + - ' results: results\n' + - ' };\n' + - '\n' + - ' return callback(null, data);\n' + - '});', + 'const results = await subscriptions.list(inputs.list.id);\n' + + 'render({ results });', hbs: '{{#each results}}\n' + '{{firstName}},{{lastName}},{{email}}\n' + @@ -246,11 +224,11 @@ export default class CUD extends Component { } async submitAndStay() { - await Form.handleChangedError(this, async () => await this.doSubmit(true)); + await this.formHandleChangedError(async () => await this.doSubmit(true)); } async submitAndLeave() { - await Form.handleChangedError(this, async () => await this.doSubmit(false)); + await this.formHandleChangedError(async () => await this.doSubmit(false)); } async doSubmit(stay) { @@ -273,6 +251,7 @@ export default class CUD extends Component { if (submitSuccessful) { if (stay) { + await this.loadFormValues(); this.enableForm(); this.setFormStatusMessage('success', t('Report template saved')); } else { diff --git a/client/src/reports/templates/List.js b/client/src/reports/templates/List.js index 40604364..a1106316 100644 --- a/client/src/reports/templates/List.js +++ b/client/src/reports/templates/List.js @@ -12,10 +12,12 @@ export default class List extends Component { render() { const t = this.props.t; - const actionLinks = [{ - label: 'Edit', - link: data => '/reports/templates/edit/' + data[0] - }]; + const actions = data => [ + { + label: 'Edit', + link: '/reports/templates/edit/' + data[0] + } + ]; const columns = [ { data: 0, title: "#" }, @@ -37,7 +39,7 @@ export default class List extends Component { {t('Report Templates')} -
+
); } diff --git a/client/src/users/List.js b/client/src/users/List.js index 3048d5b8..3da7104d 100644 --- a/client/src/users/List.js +++ b/client/src/users/List.js @@ -11,7 +11,7 @@ export default class List extends Component { render() { const t = this.props.t; - let actionLinks; + let actions; const columns = [ { data: 0, title: "#" }, @@ -21,10 +21,12 @@ export default class List extends Component { if (mailtrainConfig.isAuthMethodLocal) { columns.push({ data: 2, title: "Full Name" }); - actionLinks = [{ - label: 'Edit', - link: data => '/users/edit/' + data[0] - }]; + actions = data => [ + { + label: 'Edit', + link: '/users/edit/' + data[0] + } + ]; } return ( @@ -35,7 +37,7 @@ export default class List extends Component { {t('Users')} -
+
); } diff --git a/lib/models/reports-REMOVE.js b/lib/models/reports-REMOVE.js deleted file mode 100644 index edfe665f..00000000 --- a/lib/models/reports-REMOVE.js +++ /dev/null @@ -1,262 +0,0 @@ -'use strict'; - -const db = require('../db'); -const tableHelpers = require('../table-helpers'); -const fields = require('./fields'); -const reportTemplates = require('./report-templates'); -const tools = require('../tools'); -const _ = require('../translate')._; - -const allowedKeys = ['name', 'description', 'report_template', 'params']; - -const ReportState = { - SCHEDULED: 0, - PROCESSING: 1, - FINISHED: 2, - FAILED: 3, - MAX: 4 -}; - -module.exports.ReportState = ReportState; - -module.exports.list = (start, limit, callback) => { - tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback); -}; - -module.exports.listWithState = (state, start, limit, callback) => { - tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback); -}; - -module.exports.filter = (request, callback) => { - tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id', - ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ], - request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback); -}; - -module.exports.get = (id, callback) => { - id = Number(id) || 0; - - if (id < 1) { - return callback(new Error(_('Missing report ID'))); - } - - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - - connection.query('SELECT * FROM reports WHERE id=?', [id], (err, rows) => { - connection.release(); - if (err) { - return callback(err); - } - - if (!rows || !rows.length) { - return callback(null, false); - } - - const template = tools.convertKeys(rows[0]); - - const params = template.params.trim(); - if (params !== '') { - try { - template.paramsObject = JSON.parse(params); - } catch (err) { - return callback(err); - } - } else { - template.params = {}; - } - - return callback(null, template); - }); - }); -}; - -// This method is not supposed to be used for unsanitized inputs. It does not do any checks. -module.exports.updateFields = (id, fieldValueMap, callback) => { - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - - const clauses = []; - const values = []; - for (let key of Object.keys(fieldValueMap)) { - clauses.push(tools.toDbKey(key) + '=?'); - values.push(fieldValueMap[key]); - } - - values.push(id); - - const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1'; - connection.query(query, values, (err, result) => { - connection.release(); - if (err) { - return callback(err); - } - - return callback(null, result && result.affectedRows || false); - }); - }); -}; - -module.exports.createOrUpdate = (createMode, report, callback) => { - report = report || {}; - - const id = 'id' in report ? Number(report.id) : 0; - - if (!createMode && id < 1) { - return callback(new Error(_('Missing report ID'))); - } - - const name = (report.name || '').toString().trim(); - - if (!name) { - return callback(new Error(_('Report name must be set'))); - } - - const reportTemplateId = Number(report.reportTemplate); - reportTemplates.get(reportTemplateId, (err, reportTemplate) => { - if (err) { - return callback(err); - } - - const params = report.paramsObject; - for (const spec of reportTemplate.userFieldsObject) { - if (params[spec.id].length < spec.minOccurences) { - return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.'))); - } - - if (params[spec.id].length > spec.maxOccurences) { - return callback(new Error(_('At most ' + spec.minOccurences + ' rows in "' + spec.name + '" can be selected.'))); - } - } - - const keys = ['name', 'params']; - const values = [name, JSON.stringify(params)]; - - - Object.keys(report).forEach(key => { - let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim(); - key = tools.toDbKey(key); - - if (key === 'description') { - value = tools.purifyHTML(value); - } - - if (allowedKeys.indexOf(key) >= 0 && keys.indexOf(key) < 0) { - keys.push(key); - values.push(value); - } - }); - - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - - let query; - - if (createMode) { - query = 'INSERT INTO reports (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')'; - } else { - query = 'UPDATE reports SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1'; - values.push(id); - } - - connection.query(query, values, (err, result) => { - connection.release(); - if (err) { - return callback(err); - } - - if (createMode) { - return callback(null, result && result.insertId || false); - } else { - return callback(null, result && result.affectedRows || false); - } - }); - }); - }); -}; - -module.exports.delete = (id, callback) => { - id = Number(id) || 0; - - if (id < 1) { - return callback(new Error(_('Missing report ID'))); - } - - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - - connection.query('DELETE FROM reports WHERE id=? LIMIT 1', [id], (err, result) => { - connection.release(); - if (err) { - return callback(err); - } - - const affected = result && result.affectedRows || 0; - return callback(err, affected); - }); - }); -}; - -const campaignFieldsMapping = { - tracker_count: 'tracker.count', - country: 'tracker.country', - device_type: 'tracker.device_type', - status: 'campaign.status', - first_name: 'subscribers.first_name', - last_name: 'subscribers.last_name', - email: 'subscribers.email' -}; - -module.exports.getCampaignResults = (campaign, select, clause, callback) => { - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - - fields.list(campaign.list, (err, fieldList) => { - if (err) { - return callback(err); - } - - const fieldsMapping = fieldList.reduce((map, field) => { - map[customFieldName(field.key)] = 'subscribers.' + field.column; - return map; - }, Object.assign({}, campaignFieldsMapping)); - - let selFields = []; - for (let idx = 0; idx < select.length; idx++) { - const item = select[idx]; - if (item in fieldsMapping) { - selFields.push(fieldsMapping[item] + ' AS ' + item); - } else if (item === '*') { - selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item)); - } else { - selFields.push(item); - } - } - - const query = 'SELECT ' + selFields.join(', ') + ' FROM `subscription__' + campaign.list + '` subscribers INNER JOIN `campaign__' + campaign.id + '` campaign on subscribers.id=campaign.subscription LEFT JOIN `campaign_tracker__' + campaign.id + '` tracker on subscribers.id=tracker.subscriber ' + clause; - - connection.query(query, (err, results) => { - connection.release(); - if (err) { - return callback(err); - } - - return callback(null, results); - }); - }); - }); -}; - -function customFieldName(id) { - return id.replace(/MERGE_/, 'CUSTOM_').toLowerCase(); -} diff --git a/lib/models/subscriptions.js b/lib/models/subscriptions.js index f9b562c5..8192036e 100644 --- a/lib/models/subscriptions.js +++ b/lib/models/subscriptions.js @@ -10,14 +10,7 @@ let segments = require('./segments'); let _ = require('../translate')._; let tableHelpers = require('../table-helpers'); -const Status = { - SUBSCRIBED: 1, - UNSUBSCRIBED: 2, - BOUNCED: 3, - COMPLAINED: 4, - MAX: 5 -}; - +const Status = require('../../models/subscriptions').Status; module.exports.Status = Status; module.exports.list = (listId, start, limit, callback) => { diff --git a/lib/report-processor.js b/lib/report-processor.js index dd909278..ede02beb 100644 --- a/lib/report-processor.js +++ b/lib/report-processor.js @@ -30,7 +30,7 @@ function startWorker(report) { } try { - await reports.update(report.id, fields); + await reports.updateFields(report.id, fields); setImmediate(tryStartWorkers); } catch (err) { log.error('ReportProcessor', err); @@ -47,7 +47,7 @@ function startWorker(report) { }; try { - await reports.update(report.id, fields); + await reports.updateFields(report.id, fields); setImmediate(tryStartWorkers); } catch (err) { log.error('ReportProcessor', err); @@ -83,7 +83,7 @@ async function tryStartWorkers() { log.info('ReportProcessor', 'Starting worker'); const report = reportList[0]; - await report.updateFields(report.id, {state: reports.ReportState.PROCESSING}); + await reports.updateFields(report.id, {state: reports.ReportState.PROCESSING}); startWorker(report); } else { @@ -102,7 +102,7 @@ async function tryStartWorkers() { module.exports.start = async reportId => { if (!workers[reportId]) { log.info('ReportProcessor', 'Scheduling report id: %s', reportId); - await reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}); + await reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, last_run: null}); tryStartWorkers(); } else { log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId); @@ -124,6 +124,7 @@ module.exports.stop = async reportId => { module.exports.init = async () => { try { await reports.bulkChangeState(reports.ReportState.PROCESSING, reports.ReportState.SCHEDULED); + tryStartWorkers(); } catch (err) { log.error('ReportProcessor', err); } diff --git a/models/campaigns.js b/models/campaigns.js index 5a2b0202..fdfa3ef8 100644 --- a/models/campaigns.js +++ b/models/campaigns.js @@ -2,12 +2,22 @@ const knex = require('../lib/knex'); const dtHelpers = require('../lib/dt-helpers'); +const interoperableErrors = require('../shared/interoperable-errors'); async function listDTAjax(params) { return await dtHelpers.ajaxList(params, tx => tx('campaigns'), ['campaigns.id', 'campaigns.name', 'campaigns.description', 'campaigns.status', 'campaigns.created']); } +async function getById(id) { + const entity = await knex('campaigns').where('id', id).first(); + if (!entity) { + throw new interoperableErrors.NotFoundError(); + } + + return entity; +} module.exports = { - listDTAjax + listDTAjax, + getById }; \ No newline at end of file diff --git a/models/fields.js b/models/fields.js new file mode 100644 index 00000000..40d50404 --- /dev/null +++ b/models/fields.js @@ -0,0 +1,11 @@ +'use strict'; + +const knex = require('../lib/knex'); +const dtHelpers = require('../lib/dt-helpers'); +const interoperableErrors = require('../shared/interoperable-errors'); +const fieldsLegacy = require('../lib/models/fields'); +const bluebird = require('bluebird'); + +module.exports = { + list: bluebird.promisify(fieldsLegacy.list) +}; \ No newline at end of file diff --git a/models/lists.js b/models/lists.js index e2d85478..8b861649 100644 --- a/models/lists.js +++ b/models/lists.js @@ -2,12 +2,23 @@ const knex = require('../lib/knex'); const dtHelpers = require('../lib/dt-helpers'); +const interoperableErrors = require('../shared/interoperable-errors'); async function listDTAjax(params) { return await dtHelpers.ajaxList(params, tx => tx('lists'), ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description']); } +async function getById(id) { + const entity = await knex('lists').where('id', id).first(); + if (!entity) { + throw new interoperableErrors.NotFoundError(); + } + + return entity; +} + module.exports = { - listDTAjax + listDTAjax, + getById }; \ No newline at end of file diff --git a/models/reports.js b/models/reports.js index 3a634b7d..630cbb8b 100644 --- a/models/reports.js +++ b/models/reports.js @@ -5,24 +5,19 @@ const hasher = require('node-object-hash')(); const { enforce, filterObject } = require('../lib/helpers'); const dtHelpers = require('../lib/dt-helpers'); const interoperableErrors = require('../shared/interoperable-errors'); +const fields = require('./fields'); + +const ReportState = require('../shared/reports').ReportState; const allowedKeys = new Set(['name', 'description', 'report_template', 'params']); -const ReportState = { - SCHEDULED: 0, - PROCESSING: 1, - FINISHED: 2, - FAILED: 3, - MAX: 4 -}; - function hash(entity) { return hasher.hash(filterObject(entity, allowedKeys)); } -async function getByIdWithUserFields(id) { - const entity = await knex('reports').where('reports.id', id).innerJoin('report_templates', 'reports.report_template', 'report_templates.id').select(['reports.id', 'reports.name', 'reports.description', 'reports.report_template', 'reports.params', 'report_templates.user_fields']).first(); +async function getByIdWithTemplate(id) { + const entity = await knex('reports').where('reports.id', id).innerJoin('report_templates', 'reports.report_template', 'report_templates.id').select(['reports.id', 'reports.name', 'reports.description', 'reports.report_template', 'reports.params', 'reports.state', 'report_templates.user_fields', 'report_templates.mime_type', 'report_templates.hbs', 'report_templates.js']).first(); if (!entity) { throw new interoperableErrors.NotFoundError(); } @@ -34,10 +29,11 @@ async function getByIdWithUserFields(id) { } async function listDTAjax(params) { - return await dtHelpers.ajaxList(params, tx => tx('reports').innerJoin('report_templates', 'reports.report_template', 'report_templates.id'), ['reports.id', 'reports.name', 'report_templates.name', 'reports.description', 'reports.last_run', 'reports.state']); + return await dtHelpers.ajaxList(params, tx => tx('reports').innerJoin('report_templates', 'reports.report_template', 'report_templates.id'), ['reports.id', 'reports.name', 'report_templates.name', 'reports.description', 'reports.last_run', 'reports.state', 'report_templates.mime_type']); } async function create(entity) { + let id; await knex.transaction(async tx => { if (!await tx('report_templates').select(['id']).where('id', entity.report_template).first()) { throw new interoperableErrors.DependencyNotFoundError(); @@ -45,10 +41,12 @@ async function create(entity) { entity.params = JSON.stringify(entity.params); - const id = await tx('reports').insert(filterObject(entity, allowedKeys)); - - return id; + id = await tx('reports').insert(filterObject(entity, allowedKeys)); }); + + const reportProcessor = require('../lib/report-processor'); + await reportProcessor.start(id); + return id; } async function updateWithConsistencyCheck(entity) { @@ -71,8 +69,15 @@ async function updateWithConsistencyCheck(entity) { entity.params = JSON.stringify(entity.params); - await tx('reports').where('id', entity.id).update(filterObject(entity, allowedKeys)); + const filteredUpdates = filterObject(entity, allowedKeys); + filteredUpdates.state = ReportState.SCHEDULED; + + await tx('reports').where('id', entity.id).update(filteredUpdates); }); + + // This require is here to avoid cyclic dependency + const reportProcessor = require('../lib/report-processor'); + await reportProcessor.start(entity.id); } async function remove(id) { @@ -92,16 +97,68 @@ async function bulkChangeState(oldState, newState) { } +const campaignFieldsMapping = { + tracker_count: 'tracker.count', + country: 'tracker.country', + device_type: 'tracker.device_type', + status: 'campaign.status', + first_name: 'subscribers.first_name', + last_name: 'subscribers.last_name', + email: 'subscribers.email' +}; + +function customFieldName(id) { + return id.replace(/MERGE_/, 'CUSTOM_').toLowerCase(); +} + +async function getCampaignResults(campaign, select, extra) { + const fieldList = await fields.list(campaign.list); + + const fieldsMapping = fieldList.reduce((map, field) => { + /* Dropdowns and checkboxes are aggregated. As such, they have field.column == null and the options are in field.options. + TODO - For the time being, we ignore groupped fields. */ + if (field.column) { + map[customFieldName(field.key)] = 'subscribers.' + field.column; + } + return map; + }, Object.assign({}, campaignFieldsMapping)); + + let selFields = []; + for (let idx = 0; idx < select.length; idx++) { + const item = select[idx]; + if (item in fieldsMapping) { + selFields.push(fieldsMapping[item] + ' AS ' + item); + } else if (item === '*') { + selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item)); + } else { + selFields.push(item); + } + } + + let query = knex(`subscription__${campaign.list} AS subscribers`) + .innerJoin(`campaign__${campaign.id} AS campaign`, 'subscribers.id', 'campaign.subscription') + .leftJoin(`campaign_tracker__${campaign.id} AS tracker`, 'subscribers.id', 'tracker.subscriber') + .select(selFields); + + if (extra) { + query = extra(query); + } + + return await query; +} + + module.exports = { ReportState, hash, - getByIdWithUserFields, + getByIdWithTemplate, listDTAjax, create, updateWithConsistencyCheck, remove, updateFields, listByState, - bulkChangeState + bulkChangeState, + getCampaignResults }; \ No newline at end of file diff --git a/models/subscriptions.js b/models/subscriptions.js new file mode 100644 index 00000000..3e2ab07f --- /dev/null +++ b/models/subscriptions.js @@ -0,0 +1,23 @@ +'use strict'; + +const knex = require('../lib/knex'); +const dtHelpers = require('../lib/dt-helpers'); +const interoperableErrors = require('../shared/interoperable-errors'); + +const Status = { + SUBSCRIBED: 1, + UNSUBSCRIBED: 2, + BOUNCED: 3, + COMPLAINED: 4, + MAX: 5 +}; + +async function list(listId) { + return await knex(`subscription__${listId}`); +} + + +module.exports = { + Status, + list +}; \ No newline at end of file diff --git a/routes/reports-REMOVE.js b/routes/reports-REMOVE.js deleted file mode 100644 index fe6c5df5..00000000 --- a/routes/reports-REMOVE.js +++ /dev/null @@ -1,406 +0,0 @@ -'use strict'; - -const express = require('express'); -const passport = require('../lib/passport'); -const router = new express.Router(); -const _ = require('../lib/translate')._; -const reportTemplates = require('../lib/models/report-templates'); -const reports = require('../lib/models/reports-REMOVE'); -const reportProcessor = require('../lib/report-processor'); -const campaigns = require('../lib/models/campaigns'); -const lists = require('../lib/models/lists'); -const tools = require('../lib/tools'); -const fileHelpers = require('../lib/file-helpers'); -const util = require('util'); -const htmlescape = require('escape-html'); -const striptags = require('striptags'); -const fs = require('fs'); -const hbs = require('hbs'); - -router.all('/*', (req, res, next) => { - if (!req.user) { - req.flash('danger', _('Need to be logged in to access restricted content')); - return res.redirect('/account/login?next=' + encodeURIComponent(req.originalUrl)); - } - res.setSelectedMenu('reports'); - next(); -}); - -router.get('/', (req, res) => { - res.render('reports/reports', { - title: _('Reports') - }); -}); - - - -router.post('/ajax', (req, res) => { - reports.filter(req.body, (err, data, total, filteredTotal) => { - if (err) { - return res.json({ - error: err.message || err, - data: [] - }); - } - - res.json({ - draw: req.body.draw, - recordsTotal: total, - recordsFiltered: filteredTotal, - data: data.map((row, i) => [ - (Number(req.body.start) || 0) + 1 + i, - htmlescape(row.name || ''), - htmlescape(row.reportTemplateName || ''), - htmlescape(striptags(row.description) || ''), - getRowLastRun(row), - getRowActions(row) - ]) - }); - }); -}); - -router.get('/row/ajax/:id', (req, res) => { - respondRowActions(req.params.id, res); -}); - -router.get('/start/ajax/:id', (req, res) => { - reportProcessor.start(req.params.id, () => { - respondRowActions(req.params.id, res); - }); -}); - -router.get('/stop/ajax/:id', (req, res) => { - reportProcessor.stop(req.params.id, () => { - respondRowActions(req.params.id, res); - }); -}); - -router.get('/create', passport.csrfProtection, (req, res) => { - const reqData = req.query; - reqData.csrfToken = req.csrfToken(); - reqData.title = _('Create Report'); - reqData.useEditor = true; - - reportTemplates.quicklist((err, items) => { - if (err) { - req.flash('danger', err.message || err); - return res.redirect('/reports'); - } - - const reportTemplateId = Number(reqData.reportTemplate); - - if (reportTemplateId) { - items.forEach(item => { - if (item.id === reportTemplateId) { - item.selected = true; - } - }); - } - - reqData.reportTemplates = items; - - if (!reportTemplateId) { - res.render('reports/create-select-template', reqData); - } else { - addUserFields(reportTemplateId, reqData, null, (err, data) => { - if (err) { - req.flash('danger', err.message || err); - return res.redirect('/reports'); - } - - res.render('reports/create', data); - }); - } - }); -}); - -router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => { - const reqData = req.body; - - const reportTemplateId = Number(reqData.reportTemplate); - - addParamsObject(reportTemplateId, reqData, (err, data) => { - if (err) { - req.flash('danger', err && err.message || err || _('Could not create report')); - return res.redirect('/reports/create?' + tools.queryParams(data)); - } - - reports.createOrUpdate(true, data, (err, id) => { - if (err || !id) { - req.flash('danger', err && err.message || err || _('Could not create report')); - return res.redirect('/reports/create?' + tools.queryParams(data)); - } - - reportProcessor.start(id, () => { - req.flash('success', util.format(_('Report “%s” created'), data.name)); - res.redirect('/reports'); - }); - }); - }); -}); - -router.get('/edit/:id', passport.csrfProtection, (req, res) => { - const reqData = req.query; - reports.get(req.params.id, (err, report) => { - if (err || !report) { - req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); - return res.redirect('/reports'); - } - - report.csrfToken = req.csrfToken(); - report.title = _('Edit Report'); - report.useEditor = true; - - reportTemplates.quicklist((err, items) => { - if (err) { - req.flash('danger', err.message || err); - return res.redirect('/'); - } - - const reportTemplateId = report.reportTemplate; - - items.forEach(item => { - if (item.id === reportTemplateId) { - item.selected = true; - } - }); - - report.reportTemplates = items; - - addUserFields(reportTemplateId, reqData, report, (err, data) => { - if (err) { - req.flash('danger', err.message || err); - return res.redirect('/reports'); - } - - res.render('reports/edit', data); - }); - }); - }); -}); - -router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => { - const reqData = req.body; - const reportTemplateId = Number(reqData.reportTemplate); - - addParamsObject(reportTemplateId, reqData, (err, data) => { - if (err) { - req.flash('danger', err && err.message || err || _('Could not update report')); - return res.redirect('/reports/create?' + tools.queryParams(data)); - } - - reports.createOrUpdate(false, data, (err, updated) => { - if (err) { - req.flash('danger', err && err.message || err || _('Could not update report')); - return res.redirect('/reports/edit/' + data.id + '?' + tools.queryParams(data)); - } else if (updated) { - req.flash('success', _('Report updated')); - } else { - req.flash('info', _('Report not updated')); - } - - return res.redirect('/reports'); - }); - }); -}); - -router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => { - reports.delete(req.body.id, (err, deleted) => { - if (err) { - req.flash('danger', err && err.message || err); - } else if (deleted) { - req.flash('success', _('Report deleted')); - } else { - req.flash('info', _('Could not delete specified report')); - } - - return res.redirect('/reports'); - }); -}); - -router.get('/view/:id', (req, res) => { - reports.get(req.params.id, (err, report) => { - if (err || !report) { - req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); - return res.redirect('/reports'); - } - - reportTemplates.get(report.reportTemplate, (err, reportTemplate) => { - if (err) { - req.flash('danger', err && err.message || err || _('Could not find report template')); - return res.redirect('/reports'); - } - - if (report.state == reports.ReportState.FINISHED) { - if (reportTemplate.mimeType == 'text/html') { - - fs.readFile(fileHelpers.getReportContentFile(report), (err, reportContent) => { - if (err) { - req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); - return res.redirect('/reports'); - } - - const data = { - report: new hbs.handlebars.SafeString(reportContent), - title: report.name - }; - - res.render('reports/view', data); - }); - - } else if (reportTemplate.mimeType == 'text/csv') { - const headers = { - 'Content-Disposition': 'attachment;filename=' + fileHelpers.nameToFileName(report.name) + '.csv', - 'Content-Type': 'text/csv' - }; - - res.sendFile(fileHelpers.getReportContentFile(report), {headers: headers}); - - } else { - req.flash('danger', _('Unknown type of template')); - res.redirect('/reports'); - } - - } else { - req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); - return res.redirect('/reports'); - } - }); - }); -}); - -router.get('/output/:id', (req, res) => { - reports.get(req.params.id, (err, report) => { - if (err || !report) { - req.flash('danger', err && err.message || err || _('Could not find report with specified ID')); - return res.redirect('/reports'); - } - - fs.readFile(fileHelpers.getReportOutputFile(report), (err, output) => { - let data = { - title: 'Output for report ' + report.name - }; - - if (err) { - data.error = 'No output.'; - } else { - data.output = output; - } - - res.render('reports/output', data); - }); - }); -}); - -function getRowLastRun(row) { - return '' + (row.lastRun ? '' + row.lastRun.toISOString() + '' : '') + ''; -} - -function getRowActions(row) { - /* FIXME: add csrf protection to stop and refresh actions */ - - let requestRefresh = false; - let view, startStop; - let topic = 'data-topic-id="' + row.id + '"'; - - if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) { - view = ''; - startStop = ''; - requestRefresh = true; - - } else if (row.state == reports.ReportState.FINISHED) { - let icon = 'eye-open'; - if (row.mimeType == 'text/csv') icon = 'download-alt'; - - view = ''; - startStop = ''; - - } else if (row.state == reports.ReportState.FAILED) { - view = ''; - startStop = ''; - } - - let actions = view; - actions += ''; - actions += startStop; - actions += ''; - - return '' + - actions + - ''; -} - -function respondRowActions(id, res) { - reports.get(id, (err, report) => { - if (err) { - return res.json({ - error: err, - }); - } - - const data = {}; - data['#row-last-run-' + id] = getRowLastRun(report); - data['#row-actions-' + id] = getRowActions(report); - - res.json(data); - }); -} - - -function addUserFields(reportTemplateId, reqData, report, callback) { - reportTemplates.get(reportTemplateId, (err, reportTemplate) => { - if (err) { - return callback(err); - } - - const userFields = []; - - for (const spec of reportTemplate.userFieldsObject) { - let value = ''; - if ((spec.id + 'Selection') in reqData) { - value = reqData[spec.id + 'Selection']; - } else if (report && report.paramsObject && spec.id in report.paramsObject) { - value = report.paramsObject[spec.id].join(','); - } - - userFields.push({ - 'id': spec.id, - 'name': spec.name, - 'type': spec.type, - 'value': value, - 'isMulti': !(spec.minOccurences == 1 && spec.maxOccurences == 1) - }); - } - - const data = report ? report : reqData; - data.userFields = userFields; - - callback(null, data); - }); -} - -function addParamsObject(reportTemplateId, data, callback) { - reportTemplates.get(reportTemplateId, (err, reportTemplate) => { - if (err) { - return callback(err); - } - - const paramsObject = {}; - - for (const spec of reportTemplate.userFieldsObject) { - const sel = data[spec.id + 'Selection']; - - if (!sel) { - paramsObject[spec.id] = []; - } else { - paramsObject[spec.id] = sel.split(',').map(item => Number(item)); - } - } - - data.paramsObject = paramsObject; - - callback(null, data); - }); -} - -module.exports = router; diff --git a/routes/reports.js b/routes/reports.js new file mode 100644 index 00000000..587dc0cf --- /dev/null +++ b/routes/reports.js @@ -0,0 +1,26 @@ +'use strict'; + +const passport = require('../lib/passport'); +const _ = require('../lib/translate')._; +const reports = require('../models/reports'); +const fileHelpers = require('../lib/file-helpers'); + +const router = require('../lib/router-async').create(); + +router.getAsync('/download/:id', passport.loggedIn, async (req, res) => { + const report = await reports.getByIdWithTemplate(req.params.id); + + if (report.state == reports.ReportState.FINISHED) { + const headers = { + 'Content-Disposition': 'attachment;filename=' + fileHelpers.nameToFileName(report.name) + '.csv', + 'Content-Type': report.mime_type + }; + + res.sendFile(fileHelpers.getReportContentFile(report), {headers: headers}); + + } else { + return res.status(404).send(_('Report not found')); + } +}); + +module.exports = router; diff --git a/routes/rest/reports.js b/routes/rest/reports.js index 5d509a1d..ea1066f3 100644 --- a/routes/rest/reports.js +++ b/routes/rest/reports.js @@ -4,12 +4,13 @@ const passport = require('../../lib/passport'); const _ = require('../../lib/translate')._; const reports = require('../../models/reports'); const reportProcessor = require('../../lib/report-processor'); +const fileHelpers = require('../../lib/file-helpers'); const router = require('../../lib/router-async').create(); router.getAsync('/reports/:reportId', passport.loggedIn, async (req, res) => { - const report = await reports.getByIdWithUserFields(req.params.reportId); + const report = await reports.getByIdWithTemplate(req.params.reportId); report.hash = reports.hash(report); return res.json(report); }); @@ -38,14 +39,23 @@ router.postAsync('/reports-table', passport.loggedIn, async (req, res) => { router.postAsync('/report-start/:id', passport.loggedIn, passport.csrfProtection, async (req, res) => { await reportProcessor.start(req.params.id); - // TODO + res.json(); }); router.postAsync('/report-stop/:id', async (req, res) => { await reportProcessor.stop(req.params.id); - // TODO + res.json(); +}); + +router.getAsync('/report-content/:id', async (req, res) => { + const report = await reports.getByIdWithTemplate(req.params.id); + res.sendFile(fileHelpers.getReportContentFile(report)); +}); + +router.getAsync('/report-output/:id', async (req, res) => { + const report = await reports.getByIdWithTemplate(req.params.id); + res.sendFile(fileHelpers.getReportOutputFile(report)); }); - module.exports = router; \ No newline at end of file diff --git a/services/executor.js b/services/executor.js index 42b1294f..0632f887 100644 --- a/services/executor.js +++ b/services/executor.js @@ -129,3 +129,4 @@ process.on('message', msg => { process.send({ type: 'executor-started' }); + diff --git a/shared/reports.js b/shared/reports.js new file mode 100644 index 00000000..64b6cd05 --- /dev/null +++ b/shared/reports.js @@ -0,0 +1,13 @@ +'use strict'; + +const ReportState = { + SCHEDULED: 0, + PROCESSING: 1, + FINISHED: 2, + FAILED: 3, + MAX: 4 +}; + +module.exports = { + ReportState +}; \ No newline at end of file diff --git a/workers/reports/report-processor.js b/workers/reports/report-processor.js index ff5d3a59..b93b4584 100644 --- a/workers/reports/report-processor.js +++ b/workers/reports/report-processor.js @@ -1,10 +1,10 @@ 'use strict'; -const reports = require('../../lib/models/reports-REMOVE'); -const reportTemplates = require('../../lib/models/report-templates'); -const lists = require('../../lib/models/lists'); -const subscriptions = require('../../lib/models/subscriptions'); -const campaigns = require('../../lib/models/campaigns'); +const reports = require('../../models/reports'); +const reportTemplates = require('../../models/report-templates'); +const lists = require('../../models/lists'); +const subscriptions = require('../../models/subscriptions'); +const campaigns = require('../../models/campaigns'); const handlebars = require('handlebars'); const handlebarsHelpers = require('../../lib/handlebars-helpers'); const _ = require('../../lib/translate')._; @@ -12,136 +12,82 @@ const hbs = require('hbs'); const vm = require('vm'); const log = require('npmlog'); const fs = require('fs'); +const knex = require('../../lib/knex'); + handlebarsHelpers.registerHelpers(handlebars); -let reportId = Number(process.argv[2]); -let reportDir; - -function resolveEntities(getter, ids, callback) { - const idsRemaining = ids.slice(); - const resolved = []; - - function doWork() { - if (idsRemaining.length == 0) { - return callback(null, resolved); - } - - getter(idsRemaining.shift(), (err, entity) => { - if (err) { - return callback(err); - } - - resolved.push(entity); - return doWork(); - }); - } - - setImmediate(doWork); -} - -const userFieldTypeToGetter = { - 'campaign': (id, callback) => campaigns.get(id, false, callback), - 'list': lists.get +const userFieldGetters = { + 'campaign': campaigns.getById, + 'list': lists.getById }; -function resolveUserFields(userFields, params, callback) { - const userFieldsRemaining = userFields.slice(); - const resolved = {}; - function doWork() { - if (userFieldsRemaining.length == 0) { - return callback(null, resolved); - } +async function main() { + try { + const reportId = Number(process.argv[2]); - const spec = userFieldsRemaining.shift(); - const getter = userFieldTypeToGetter[spec.type]; + const report = await reports.getByIdWithTemplate(reportId); - if (getter) { - return resolveEntities(getter, params[spec.id], (err, entities) => { - if (spec.minOccurences == 1 && spec.maxOccurences == 1) { - resolved[spec.id] = entities[0]; - } else { - resolved[spec.id] = entities; - } + const inputs = {}; - doWork(); - }); - } else { - return callback(new Error(_('Unknown user field type "' + spec.type + '".'))); - } - } - - setImmediate(doWork); -} - -function doneSuccess() { - process.exit(0); -} - -function doneFail() { - process.exit(1) -} - - - -reports.get(reportId, (err, report) => { - if (err || !report) { - log.error('reports', err && err.message || err || _('Could not find report with specified ID')); - doneFail(); - } - - reportTemplates.get(report.reportTemplate, (err, reportTemplate) => { - if (err) { - log.error('reports', err && err.message || err || _('Could not find report template')); - doneFail(); - } - - resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => { - if (err) { - log.error('reports', err.message || err); - doneFail(); + for (const spec of report.user_fields) { + const getter = userFieldGetters[spec.type]; + if (!getter) { + throw new Error(_('Unknown user field type "' + spec.type + '".')); } - const campaignsProxy = { - results: reports.getCampaignResults, - list: campaigns.list, - get: campaigns.get - }; - - const subscriptionsProxy = { - list: subscriptions.list - }; - - const sandbox = { - console, - campaigns: campaignsProxy, - subscriptions: subscriptionsProxy, - inputs, - - callback: (err, outputs) => { - if (err) { - log.error('reports', err.message || err); - doneFail(); - } - - const hbsTmpl = handlebars.compile(reportTemplate.hbs); - const reportText = hbsTmpl(outputs); - - process.stdout.write(reportText); - doneSuccess(); - } - }; - - const script = new vm.Script(reportTemplate.js); - - try { - script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000}); - } catch (err) { - console.error(err); - doneFail(); + const entities = []; + for (const id of report.params[spec.id]) { + entities.push(await getter(id)); } - }); - }); -}); + + if (spec.minOccurences == 1 && spec.maxOccurences == 1) { + inputs[spec.id] = entities[0]; + } else { + inputs[spec.id] = entities; + } + } + + const campaignsProxy = { + getResults: reports.getCampaignResults, + getById: campaigns.getById + }; + + const subscriptionsProxy = { + list: subscriptions.list + }; + + const sandbox = { + console, + campaigns: campaignsProxy, + subscriptions: subscriptionsProxy, + knex, + process, + inputs, + + render: data => { + const hbsTmpl = handlebars.compile(report.hbs); + const reportText = hbsTmpl(data); + + process.stdout.write(reportText); + } + }; + + const js = + '(async function() {' + + report.js + + '})().then(() => process.exit(0)).catch(err => { console.error(err); process.exit(1); })'; + + const script = new vm.Script(js); + + script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000}); + + } catch (err) { + console.error(err); + process.exit(1); + } +} + +main();