diff --git a/client/src/campaigns/List.js b/client/src/campaigns/List.js index 0fda63aa..745b67fa 100644 --- a/client/src/campaigns/List.js +++ b/client/src/campaigns/List.js @@ -2,38 +2,17 @@ import React, {Component} from 'react'; import {withTranslation} from '../lib/i18n'; -import { - ButtonDropdown, - Icon -} from '../lib/bootstrap-components'; -import { - DropdownLink, - NavDropdown, - requiresAuthenticatedUser, - Title, - Toolbar, - withPageHelpers -} from '../lib/page'; -import { - withAsyncErrorHandler, - withErrorHandling -} from '../lib/error-handling'; +import {ButtonDropdown, Icon} from '../lib/bootstrap-components'; +import {DropdownLink, requiresAuthenticatedUser, Title, Toolbar, withPageHelpers} from '../lib/page'; +import {withAsyncErrorHandler, withErrorHandling} from '../lib/error-handling'; import {Table} from '../lib/table'; -import moment - from 'moment'; -import { - CampaignSource, - CampaignStatus, - CampaignType -} from "../../../shared/campaigns"; +import moment from 'moment'; +import {CampaignSource, CampaignStatus, CampaignType} from "../../../shared/campaigns"; import {checkPermissions} from "../lib/permissions"; import {getCampaignLabels} from "./helpers"; -import { - tableAddDeleteButton, - tableRestActionDialogInit, - tableRestActionDialogRender -} from "../lib/modals"; +import {tableAddDeleteButton, tableRestActionDialogInit, tableRestActionDialogRender} from "../lib/modals"; import {withComponentMixins} from "../lib/decorator-helpers"; +import styles from "./styles.scss"; @withComponentMixins([ withTranslation, @@ -79,7 +58,7 @@ export default class List extends Component { const columns = [ { data: 1, title: t('name') }, - { data: 2, title: t('id'), render: data => {data} }, + { data: 2, title: t('id'), render: data => {data}, className: styles.tblCol_id }, { data: 3, title: t('description') }, { data: 4, title: t('type'), render: data => this.campaignTypeLabels[data] }, { @@ -101,6 +80,7 @@ export default class List extends Component { { data: 8, title: t('created'), render: data => moment(data).fromNow() }, { data: 9, title: t('namespace') }, { + className: styles.tblCol_buttons, actions: data => { const actions = []; const perms = data[10]; diff --git a/client/src/campaigns/Statistics.js b/client/src/campaigns/Statistics.js index 2265638d..0abad10a 100644 --- a/client/src/campaigns/Statistics.js +++ b/client/src/campaigns/Statistics.js @@ -1,26 +1,17 @@ 'use strict'; import React, {Component} from 'react'; -import PropTypes - from 'prop-types'; +import PropTypes from 'prop-types'; import {withTranslation} from '../lib/i18n'; -import { - requiresAuthenticatedUser, - Title, - withPageHelpers -} from '../lib/page'; -import { - withAsyncErrorHandler, - withErrorHandling -} from '../lib/error-handling'; -import axios - from "../lib/axios"; +import {Trans} from 'react-i18next'; +import {requiresAuthenticatedUser, Title, withPageHelpers} from '../lib/page'; +import {withAsyncErrorHandler, withErrorHandling} from '../lib/error-handling'; +import axios from "../lib/axios"; import {getUrl} from "../lib/urls"; import {AlignedRow} from "../lib/form"; import {Icon} from "../lib/bootstrap-components"; -import styles - from "./styles.scss"; +import styles from "./styles.scss"; import {Link} from "react-router-dom"; import {withComponentMixins} from "../lib/decorator-helpers"; @@ -128,6 +119,14 @@ export default class Statistics extends Component { {renderMetricsWithProgress('unsubscribed', t('unsubscribed'), 'warning')} {!entity.open_tracking_disabled && renderMetricsWithProgress('opened', t('opened'), 'success')} {!entity.click_tracking_disabled && renderMetricsWithProgress('clicks', t('clicked'), 'success')} + +
+ +

{t('Quick Reports')}

+ Below, you can download pre-made reports related to this campaign. Each link generates a CSV file that can be viewed in a spreadsheet editor. Custom reports and reports that cover more than one campaign can be created through Reports functionality of Mailtrain. + ); } diff --git a/client/src/campaigns/styles.scss b/client/src/campaigns/styles.scss index c8b29d41..a7270d50 100644 --- a/client/src/campaigns/styles.scss +++ b/client/src/campaigns/styles.scss @@ -92,3 +92,12 @@ .overrideCheckbox{ margin-top: -8px !important; } + +.tblCol_id { + min-width: 5ex; + max-width: 8ex; +} + +.tblCol_buttons { + min-width: 5.8rem; +} \ No newline at end of file diff --git a/client/src/lib/table.js b/client/src/lib/table.js index f613c6a9..e51933ae 100644 --- a/client/src/lib/table.js +++ b/client/src/lib/table.js @@ -276,6 +276,7 @@ class Table extends Component { const dtOptions = { columns, + autoWidth: false, pageLength: this.props.pageLength, dom: // This overrides Bootstrap 4 settings. It may need to be updated if there are updates in the DataTables Bootstrap 4 plugin. "<'row'<'col-sm-12 col-md-6'l><'col-sm-12 col-md-6'f>>" + diff --git a/client/src/reports/templates/CUD.js b/client/src/reports/templates/CUD.js index 2c5f80bd..eae6c732 100644 --- a/client/src/reports/templates/CUD.js +++ b/client/src/reports/templates/CUD.js @@ -163,8 +163,8 @@ export default class CUD extends Component { js: 'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["field:country", "count_opened", "count_all"], (query, col) =>\n' + ' query.count("* AS count_all")\n' + - ' .select(knex.raw("SUM(IF(`" + col(tracker:count) +"` IS NULL, 0, 1)) AS count_opened"))\n' + - ' .groupBy("field:country")\n' + + ' .select(knex.raw("SUM(IF(`" + col("tracker:count") +"` IS NULL, 0, 1)) AS count_opened"))\n' + + ' .groupBy(col("field:country"))\n' + ')\n' + '\n' + 'for (const row of results) {\n' + diff --git a/server/app-builder.js b/server/app-builder.js index 3203643b..42105d55 100644 --- a/server/app-builder.js +++ b/server/app-builder.js @@ -21,6 +21,7 @@ const api = require('./routes/api'); // These are routes for the new React-based client const reports = require('./routes/reports'); +const quickReports = require('./routes/quick-reports'); const subscriptions = require('./routes/subscriptions'); const subscription = require('./routes/subscription'); const sandboxedMosaico = require('./routes/sandboxed-mosaico'); @@ -286,6 +287,8 @@ async function createApp(appType) { useWith404Fallback('/rpts', reports); // This needs to be different from "reports", which is already used by the UI } + useWith404Fallback('/quick-rpts', quickReports); + // API endpoints useWith404Fallback('/api', api); diff --git a/server/lib/report-helpers.js b/server/lib/report-helpers.js index b2d3da43..92725c2f 100644 --- a/server/lib/report-helpers.js +++ b/server/lib/report-helpers.js @@ -1,6 +1,8 @@ 'use strict'; const path = require('path'); +const csvStringify = require('csv-stringify'); +const stream = require('stream'); function nameToFileName(name) { return name. @@ -25,10 +27,51 @@ function getReportOutputFile(report) { return getReportFileBase(report) + '.err'; } +async function renderCsvFromStream(readable, writable, opts, transform) { + const finished = new Promise((success, fail) => { + let lastReadable = readable; + + const stringifier = csvStringify(opts); + + stringifier.on('finish', () => success()); + stringifier.on('error', err => fail(err)); + + if (transform) { + const rowTransform = new stream.Transform({ + objectMode: true, + transform(row, encoding, callback) { + async function performTransform() { + try { + const newRow = await transform(row, encoding); + callback(null, newRow); + } catch (err) { + callback(err); + } + } + + // noinspection JSIgnoredPromiseFromCall + performTransform(); + } + }); + + lastReadable.on('error', err => fail(err)); + lastReadable.pipe(rowTransform); + + lastReadable = rowTransform; + } + + stringifier.pipe(writable); + lastReadable.pipe(stringifier); + }); + + await finished; +} + module.exports = { getReportContentFile, getReportOutputFile, nameToFileName, - reportFilesDir + reportFilesDir, + renderCsvFromStream }; diff --git a/server/models/reports.js b/server/models/reports.js index 992ae234..0f2bde6c 100644 --- a/server/models/reports.js +++ b/server/models/reports.js @@ -148,10 +148,8 @@ async function bulkChangeState(oldState, newState) { return await knex('reports').where('state', oldState).update('state', newState); } -async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, asStream) { - const subsQrys = []; - - const commonFieldsMapping = {}; +async function getCampaignCommonListFields(campaign) { + const listFields = {}; let firstIteration = true; for (const cpgList of campaign.lists) { const cpgListId = cpgList.list; @@ -165,16 +163,122 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */ if (fld.column) { const fldKey = 'field:' + fld.key.toLowerCase(); - if (firstIteration || commonFieldsMapping[fldKey]) { - commonFieldsMapping[fldKey] = 'subscriptions.' + fld.column; + + if (firstIteration) { + listFields[fldKey] = { + key: fld.key, + name: fld.name, + description: fld.description + }; + } + + if (fldKey in listFields) { assignedFlds.add(fldKey); } } } - for (const fldKey in commonFieldsMapping) { + for (const fldKey in listFields) { if (!assignedFlds.has(fldKey)) { - delete commonFieldsMapping[fldKey]; + delete listFields[fldKey]; + } + } + + firstIteration = false; + } + + return listFields; +} + +async function _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, asStream) { + const subsQrys = []; + joins = joins || []; + + const knexJoinFns = []; + + const commonFieldsMapping = { + 'subscription:status': 'subscriptions.status', + 'subscription:id': 'subscriptions.id', + 'subscription:cid': 'subscriptions.cid', + 'subscription:email': 'subscriptions.email' + }; + + for (const join of joins) { + const prefix = join.prefix; + const type = join.type; + const onConditions = join.onConditions || {}; + + const getConds = (alias, cpgListId) => { + const conds = { + [alias + '.campaign']: knex.raw('?', [campaign.id]), + [alias + '.list']: knex.raw('?', [cpgListId]), + [alias + '.subscription']: 'subscriptions.id', + }; + + for (const onConditionKey in onConditions) { + conds[alias + '.' + onConditionKey] = onConditions[onConditionKey]; + } + + return conds; + }; + + if (type === 'messages') { + const alias = 'campaign_messages_' + prefix; + + commonFieldsMapping[`${prefix}:status`] = alias + '.status'; + + knexJoinFns.push((qry, cpgListId) => qry.leftJoin('campaign_messages AS ' + alias, getConds(alias, cpgListId))); + + } else if (type === 'links') { + const alias = 'campaign_links_' + prefix; + + commonFieldsMapping[`${prefix}:count`] = {raw: 'COALESCE(`' + alias + '`.`count`, 0)'}; + commonFieldsMapping[`${prefix}:link`] = alias + '.link'; + commonFieldsMapping[`${prefix}:country`] = alias + '.country'; + commonFieldsMapping[`${prefix}:deviceType`] = alias + '.device_type'; + + knexJoinFns.push((qry, cpgListId) => qry.leftJoin('campaign_links AS ' + alias, getConds(alias, cpgListId))); + + } else { + throw new Error(`Unknown join type "${type}"`); + } + } + + + const listsFields = {}; + const permittedListFields = new Set(); + let firstIteration = true; + for (const cpgList of campaign.lists) { + const cpgListId = cpgList.list; + + const listFields = {}; + listsFields[cpgListId] = listFields; + + const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId); + + const assignedFlds = new Set(); + + for (const fld of flds) { + /* Dropdown and checkbox groups have field.column == null + For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */ + if (fld.column) { + const fldKey = 'field:' + fld.key.toLowerCase(); + + listFields[fldKey] = 'subscriptions.' + fld.column; + + if (firstIteration) { + permittedListFields.add(fldKey); + } + + if (permittedListFields.has(fldKey)) { + assignedFlds.add(fldKey); + } + } + } + + for (const fldKey in [...permittedListFields]) { + if (!assignedFlds.has(fldKey)) { + permittedListFields.delete(fldKey); } } @@ -183,36 +287,29 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a for (const cpgList of campaign.lists) { const cpgListId = cpgList.list; + const listFields = listsFields[cpgListId]; - const campaignFieldsMapping = { - 'list:id': {raw: knex.raw('?', [cpgListId])}, - 'tracker:count': {raw: 'COALESCE(`campaign_links`.`count`, 0)'}, - 'tracker:country': 'campaign_links.country', - 'tracker:deviceType': 'campaign_links.device_type', - 'tracker:status': 'campaign_messages.status', - 'subscription:status': 'subscriptions.status', - 'subscription:id': 'subscriptions.id', - 'subscription:cid': 'subscriptions.cid', - 'subscription:email': 'subscriptions.email' - }; + for (const fldKey in listFields) { + if (!permittedListFields.has(fldKey)) { + delete listFields[fldKey]; + } + } + } + + + for (const cpgList of campaign.lists) { + const cpgListId = cpgList.list; const fieldsMapping = { ...commonFieldsMapping, - ...campaignFieldsMapping + ...listsFields[cpgListId], + 'list:id': {raw: knex.raw('?', [cpgListId])} }; - const getColIdIfExists = (colId, getter) => { - if (colId in fieldsMapping) { - return getter(colId); - } else { - throw new Error(`Unknown column id ${colId}`); - } - } - const getSelField = item => { const itemMapping = fieldsMapping[item]; if (typeof itemMapping === 'string') { - return fieldsMapping[item] + ' AS ' + item; + return itemMapping + ' AS ' + item; } else if (itemMapping.raw) { return knex.raw(fieldsMapping[item].raw + ' AS `' + item + '`'); } @@ -230,23 +327,22 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a } } - let query = knex(`subscription__${cpgListId} AS subscriptions`) - .leftJoin('campaign_messages', { - 'campaign_messages.campaign': knex.raw('?', [campaign.id]), - 'campaign_messages.list': knex.raw('?', [cpgListId]), - 'campaign_messages.subscription': 'subscriptions.id' - }) - .leftJoin('campaign_links', { - 'campaign_links.campaign': knex.raw('?', [campaign.id]), - 'campaign_links.list': knex.raw('?', [cpgListId]), - 'campaign_links.subscription': 'subscriptions.id' - }) - .select(selFields); + let query = knex(`subscription__${cpgListId} AS subscriptions`).select(selFields); + + for (const knexJoinFn of knexJoinFns) { + query = knexJoinFn(query, cpgListId); + } if (listQryFn) { query = listQryFn( query, - colId => getColIdIfExists(colId, x => fieldsMapping[x]) + colId => { + if (colId in fieldsMapping) { + return fieldsMapping[colId]; + } else { + throw new Error(`Unknown column id ${colId}`); + } + } ); } @@ -256,13 +352,21 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a if (subsQrys.length > 0) { let subsSql, subsBindings; + const fieldsSet = new Set([...Object.keys(commonFieldsMapping), ...permittedListFields, 'list:id']); + const applyUnionQryFn = (subsSql, subsBindings) => { if (unionQryFn) { return unionQryFn( knex.from(function() { return knex.raw('(' + subsSql + ')', subsBindings); }), - colId => getColIdIfExists(colId, x => x) + colId => { + if (fieldsSet.has(colId)) { + return colId; + } else { + throw new Error(`Unknown column id ${colId}`); + } + } ); } else { return knex.raw(subsSql, subsBindings); @@ -311,10 +415,11 @@ async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryF return await _getCampaignStatistics( campaign, select, + [{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}], unionQryFn, (qry, col) => listQryFn( qry.where(function() { - this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN) + this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), LinkId.OPEN) }), col ), @@ -330,10 +435,11 @@ async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQry return await _getCampaignStatistics( campaign, select, + [{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}], unionQryFn, (qry, col) => listQryFn( qry.where(function() { - this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK) + this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), LinkId.GENERAL_CLICK) }), col ), @@ -349,10 +455,11 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis return await _getCampaignStatistics( campaign, select, + [{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}], unionQryFn, (qry, col) => listQryFn( qry.where(function() { - this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK) + this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), '>', LinkId.GENERAL_CLICK) }), col ), @@ -360,12 +467,12 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis ); } -async function getCampaignStatistics(campaign, select, unionQryFn, listQryFn) { - return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, false); +async function getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn) { + return await _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, false); } -async function getCampaignStatisticsStream(campaign, select, unionQryFn, listQryFn) { - return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, true); +async function getCampaignStatisticsStream(campaign, select, joins, unionQryFn, listQryFn) { + return await _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, true); } async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) { @@ -405,6 +512,7 @@ module.exports.remove = remove; module.exports.updateFields = updateFields; module.exports.listByState = listByState; module.exports.bulkChangeState = bulkChangeState; +module.exports.getCampaignCommonListFields = getCampaignCommonListFields; module.exports.getCampaignStatistics = getCampaignStatistics; module.exports.getCampaignStatisticsStream = getCampaignStatisticsStream; module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics; diff --git a/server/routes/quick-reports.js b/server/routes/quick-reports.js new file mode 100644 index 00000000..d816697d --- /dev/null +++ b/server/routes/quick-reports.js @@ -0,0 +1,59 @@ +'use strict'; + +const passport = require('../lib/passport'); +const shares = require('../models/shares'); +const contextHelpers = require('../lib/context-helpers'); +const {renderCsvFromStream} = require('../lib/report-helpers'); +const reports = require('../models/reports'); +const campaigns = require('../models/campaigns'); +const {castToInteger} = require('../lib/helpers'); +const {SubscriptionStatus} = require('../../shared/lists'); +const knex = require('../lib/knex'); +const {LinkId} = require('../models/links'); + +const router = require('../lib/router-async').create(); + +router.getAsync('/open-and-click-counts/:campaignId', passport.loggedIn, async (req, res) => { + const campaignId = castToInteger(req.params.campaignId); + + await shares.enforceEntityPermission(req.context, 'campaign', campaignId, 'viewStats'); + const campaign = await campaigns.getById(req.context, campaignId, false); + + const listFields = await reports.getCampaignCommonListFields(campaign); + + const results = await reports.getCampaignStatisticsStream( + campaign, + ['subscription:email', 'open_tracker:count', 'click_tracker:count', 'open_tracker:country', ...Object.keys(listFields)], + [ + {type: 'links', prefix: 'open_tracker', onConditions: {link: knex.raw('?', [LinkId.OPEN])} }, + {type: 'links', prefix: 'click_tracker', onConditions: {link: knex.raw('?', [LinkId.GENERAL_CLICK])} } + ], + null, + (qry, col) => qry + .where(col('subscription:status'), SubscriptionStatus.SUBSCRIBED) + ); + + res.set({ + 'Content-Disposition': `attachment;filename=campaign-open-and-click-counts-${campaign.cid}.csv`, + 'Content-Type': 'text/csv' + }); + + await renderCsvFromStream( + results, + res, + { + header: true, + columns: [ + { key: 'subscription:email', header: 'Email' }, + { key: 'open_tracker:count', header: 'Open count' }, + { key: 'click_tracker:count', header: 'Click count' }, + { key: 'open_tracker:country', header: 'Country (first open)' }, + ...Object.keys(listFields).map(key => ({key, header: listFields[key].key})) + ], + delimiter: ',' + }, + async (row, encoding) => row + ); +}); + +module.exports = router; diff --git a/server/routes/reports.js b/server/routes/reports.js index bd1f4d09..d61fbc59 100644 --- a/server/routes/reports.js +++ b/server/routes/reports.js @@ -5,6 +5,7 @@ const reports = require('../models/reports'); const reportHelpers = require('../lib/report-helpers'); const shares = require('../models/shares'); const contextHelpers = require('../lib/context-helpers'); +const {castToInteger} = require('../lib/helpers'); const router = require('../lib/router-async').create(); @@ -14,9 +15,10 @@ const fileSuffixes = { }; router.getAsync('/:id/download', passport.loggedIn, async (req, res) => { - await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent'); + const reportId = castToInteger(req.params.id); + await shares.enforceEntityPermission(req.context, 'report', reportId, 'viewContent'); - const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id, false); + const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), reportId, false); if (report.state == reports.ReportState.FINISHED) { const headers = { diff --git a/server/services/workers/reports/report-processor.js b/server/services/workers/reports/report-processor.js index b7883032..80c48134 100644 --- a/server/services/workers/reports/report-processor.js +++ b/server/services/workers/reports/report-processor.js @@ -6,14 +6,11 @@ const subscriptions = require('../../../models/subscriptions'); const { SubscriptionSource, SubscriptionStatus } = require('../../../../shared/lists'); const campaigns = require('../../../models/campaigns'); const handlebars = require('handlebars'); -const hbs = require('hbs'); const vm = require('vm'); const log = require('../../../lib/log'); -const fs = require('fs'); const knex = require('../../../lib/knex'); const contextHelpers = require('../../../lib/context-helpers'); - -const csvStringify = require('csv-stringify'); +const {renderCsvFromStream} = require('../../../lib/report-helpers'); const stream = require('stream'); async function main() { @@ -75,46 +72,7 @@ async function main() { inputs, SubscriptionSource, SubscriptionStatus, - - renderCsvFromStream: async (readable, opts, transform) => { - const finished = new Promise((success, fail) => { - let lastReadable = readable; - - const stringifier = csvStringify(opts); - - stringifier.on('finish', () => success()); - stringifier.on('error', err => fail(err)); - - if (transform) { - const rowTransform = new stream.Transform({ - objectMode: true, - transform(row, encoding, callback) { - async function performTransform() { - try { - const newRow = await transform(row, encoding); - callback(null, newRow); - } catch (err) { - callback(err); - } - } - - // noinspection JSIgnoredPromiseFromCall - performTransform(); - } - }); - - lastReadable.on('error', err => fail(err)); - lastReadable.pipe(rowTransform); - - lastReadable = rowTransform; - } - - stringifier.pipe(process.stdout); - lastReadable.pipe(stringifier); - }); - - await finished; - }, + renderCsvFromStream: (readable, opts, transform) => renderCsvFromStream(readable, process.stdout, opts, transform), render: data => { const hbsTmpl = handlebars.compile(report.hbs);