public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Added quick reports (at this moment only one) to campaign statistics page.

Browse source
This commit is contained in:
Tomas Bures 2019-04-22 22:46:48 +02:00
parent 3e3c3a24fe
commit 72ffe065d2
11 changed files with 305 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

38
client/src/campaigns/List.js
View file

@ -2,38 +2,17 @@
import React, {Component} from 'react';
import {withTranslation} from '../lib/i18n';
import {
ButtonDropdown,
Icon
} from '../lib/bootstrap-components';
import {
DropdownLink,
NavDropdown,
requiresAuthenticatedUser,
Title,
Toolbar,
withPageHelpers
} from '../lib/page';
import {
withAsyncErrorHandler,
withErrorHandling
} from '../lib/error-handling';
import {ButtonDropdown, Icon} from '../lib/bootstrap-components';
import {DropdownLink, requiresAuthenticatedUser, Title, Toolbar, withPageHelpers} from '../lib/page';
import {withAsyncErrorHandler, withErrorHandling} from '../lib/error-handling';
import {Table} from '../lib/table';
import moment
from 'moment';
import {
CampaignSource,
CampaignStatus,
CampaignType
} from "../../../shared/campaigns";
import moment from 'moment';
import {CampaignSource, CampaignStatus, CampaignType} from "../../../shared/campaigns";
import {checkPermissions} from "../lib/permissions";
import {getCampaignLabels} from "./helpers";
import {
tableAddDeleteButton,
tableRestActionDialogInit,
tableRestActionDialogRender
} from "../lib/modals";
import {tableAddDeleteButton, tableRestActionDialogInit, tableRestActionDialogRender} from "../lib/modals";
import {withComponentMixins} from "../lib/decorator-helpers";
import styles from "./styles.scss";
@withComponentMixins([
withTranslation,
@ -79,7 +58,7 @@ export default class List extends Component {
const columns = [
{ data: 1, title: t('name') },
{ data: 2, title: t('id'), render: data => <code>{data}</code> },
{ data: 2, title: t('id'), render: data => <code>{data}</code>, className: styles.tblCol_id },
{ data: 3, title: t('description') },
{ data: 4, title: t('type'), render: data => this.campaignTypeLabels[data] },
{
@ -101,6 +80,7 @@ export default class List extends Component {
{ data: 8, title: t('created'), render: data => moment(data).fromNow() },
{ data: 9, title: t('namespace') },
{
className: styles.tblCol_buttons,
actions: data => {
const actions = [];
const perms = data[10];

29
client/src/campaigns/Statistics.js
View file

@ -1,26 +1,17 @@
'use strict';
import React, {Component} from 'react';
import PropTypes
from 'prop-types';
import PropTypes from 'prop-types';
import {withTranslation} from '../lib/i18n';
import {
requiresAuthenticatedUser,
Title,
withPageHelpers
} from '../lib/page';
import {
withAsyncErrorHandler,
withErrorHandling
} from '../lib/error-handling';
import axios
from "../lib/axios";
import {Trans} from 'react-i18next';
import {requiresAuthenticatedUser, Title, withPageHelpers} from '../lib/page';
import {withAsyncErrorHandler, withErrorHandling} from '../lib/error-handling';
import axios from "../lib/axios";
import {getUrl} from "../lib/urls";
import {AlignedRow} from "../lib/form";
import {Icon} from "../lib/bootstrap-components";
import styles
from "./styles.scss";
import styles from "./styles.scss";
import {Link} from "react-router-dom";
import {withComponentMixins} from "../lib/decorator-helpers";
@ -128,6 +119,14 @@ export default class Statistics extends Component {
{renderMetricsWithProgress('unsubscribed', t('unsubscribed'), 'warning')}
{!entity.open_tracking_disabled && renderMetricsWithProgress('opened', t('opened'), 'success')}
{!entity.click_tracking_disabled && renderMetricsWithProgress('clicks', t('clicked'), 'success')}
<hr/>
<h3>{t('Quick Reports')}</h3>
<small className="text-muted"><Trans>Below, you can download pre-made reports related to this campaign. Each link generates a CSV file that can be viewed in a spreadsheet editor. Custom reports and reports that cover more than one campaign can be created through <Link to="/reports">Reports</Link> functionality of Mailtrain.</Trans></small>
<ul className="list-unstyled my-3">
<li><a href={getUrl(`quick-rpts/open-and-click-counts/${entity.id}`)}>Open and click counts per currently subscribed subscriber</a></li>
</ul>
</div>
);
}

9
client/src/campaigns/styles.scss
View file

@ -92,3 +92,12 @@
.overrideCheckbox{
margin-top: -8px !important;
}
.tblCol_id {
min-width: 5ex;
max-width: 8ex;
}
.tblCol_buttons {
min-width: 5.8rem;
}

1
client/src/lib/table.js
View file

@ -276,6 +276,7 @@ class Table extends Component {
const dtOptions = {
columns,
autoWidth: false,
pageLength: this.props.pageLength,
dom: // This overrides Bootstrap 4 settings. It may need to be updated if there are updates in the DataTables Bootstrap 4 plugin.
"<'row'<'col-sm-12 col-md-6'l><'col-sm-12 col-md-6'f>>" +

4
client/src/reports/templates/CUD.js
View file

@ -163,8 +163,8 @@ export default class CUD extends Component {
js:
'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["field:country", "count_opened", "count_all"], (query, col) =>\n' +
' query.count("* AS count_all")\n' +
' .select(knex.raw("SUM(IF(`" + col(tracker:count) +"` IS NULL, 0, 1)) AS count_opened"))\n' +
' .groupBy("field:country")\n' +
' .select(knex.raw("SUM(IF(`" + col("tracker:count") +"` IS NULL, 0, 1)) AS count_opened"))\n' +
' .groupBy(col("field:country"))\n' +
')\n' +
'\n' +
'for (const row of results) {\n' +

3
server/app-builder.js
View file

@ -21,6 +21,7 @@ const api = require('./routes/api');
// These are routes for the new React-based client
const reports = require('./routes/reports');
const quickReports = require('./routes/quick-reports');
const subscriptions = require('./routes/subscriptions');
const subscription = require('./routes/subscription');
const sandboxedMosaico = require('./routes/sandboxed-mosaico');
@ -286,6 +287,8 @@ async function createApp(appType) {
useWith404Fallback('/rpts', reports); // This needs to be different from "reports", which is already used by the UI
}
useWith404Fallback('/quick-rpts', quickReports);
// API endpoints
useWith404Fallback('/api', api);

45
server/lib/report-helpers.js
View file

@ -1,6 +1,8 @@
'use strict';
const path = require('path');
const csvStringify = require('csv-stringify');
const stream = require('stream');
function nameToFileName(name) {
return name.
@ -25,10 +27,51 @@ function getReportOutputFile(report) {
return getReportFileBase(report) + '.err';
}
async function renderCsvFromStream(readable, writable, opts, transform) {
const finished = new Promise((success, fail) => {
let lastReadable = readable;
const stringifier = csvStringify(opts);
stringifier.on('finish', () => success());
stringifier.on('error', err => fail(err));
if (transform) {
const rowTransform = new stream.Transform({
objectMode: true,
transform(row, encoding, callback) {
async function performTransform() {
try {
const newRow = await transform(row, encoding);
callback(null, newRow);
} catch (err) {
callback(err);
}
}
// noinspection JSIgnoredPromiseFromCall
performTransform();
}
});
lastReadable.on('error', err => fail(err));
lastReadable.pipe(rowTransform);
lastReadable = rowTransform;
}
stringifier.pipe(writable);
lastReadable.pipe(stringifier);
});
await finished;
}
module.exports = {
getReportContentFile,
getReportOutputFile,
nameToFileName,
reportFilesDir
reportFilesDir,
renderCsvFromStream
};

208
server/models/reports.js
View file

@ -148,10 +148,8 @@ async function bulkChangeState(oldState, newState) {
return await knex('reports').where('state', oldState).update('state', newState);
}
async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, asStream) {
const subsQrys = [];
const commonFieldsMapping = {};
async function getCampaignCommonListFields(campaign) {
const listFields = {};
let firstIteration = true;
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
@ -165,16 +163,122 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
if (fld.column) {
const fldKey = 'field:' + fld.key.toLowerCase();
if (firstIteration || commonFieldsMapping[fldKey]) {
commonFieldsMapping[fldKey] = 'subscriptions.' + fld.column;
if (firstIteration) {
listFields[fldKey] = {
key: fld.key,
name: fld.name,
description: fld.description
};
}
if (fldKey in listFields) {
assignedFlds.add(fldKey);
}
}
}
for (const fldKey in commonFieldsMapping) {
for (const fldKey in listFields) {
if (!assignedFlds.has(fldKey)) {
delete commonFieldsMapping[fldKey];
delete listFields[fldKey];
}
}
firstIteration = false;
}
return listFields;
}
async function _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, asStream) {
const subsQrys = [];
joins = joins || [];
const knexJoinFns = [];
const commonFieldsMapping = {
'subscription:status': 'subscriptions.status',
'subscription:id': 'subscriptions.id',
'subscription:cid': 'subscriptions.cid',
'subscription:email': 'subscriptions.email'
};
for (const join of joins) {
const prefix = join.prefix;
const type = join.type;
const onConditions = join.onConditions || {};
const getConds = (alias, cpgListId) => {
const conds = {
[alias + '.campaign']: knex.raw('?', [campaign.id]),
[alias + '.list']: knex.raw('?', [cpgListId]),
[alias + '.subscription']: 'subscriptions.id',
};
for (const onConditionKey in onConditions) {
conds[alias + '.' + onConditionKey] = onConditions[onConditionKey];
}
return conds;
};
if (type === 'messages') {
const alias = 'campaign_messages_' + prefix;
commonFieldsMapping[`${prefix}:status`] = alias + '.status';
knexJoinFns.push((qry, cpgListId) => qry.leftJoin('campaign_messages AS ' + alias, getConds(alias, cpgListId)));
} else if (type === 'links') {
const alias = 'campaign_links_' + prefix;
commonFieldsMapping[`${prefix}:count`] = {raw: 'COALESCE(`' + alias + '`.`count`, 0)'};
commonFieldsMapping[`${prefix}:link`] = alias + '.link';
commonFieldsMapping[`${prefix}:country`] = alias + '.country';
commonFieldsMapping[`${prefix}:deviceType`] = alias + '.device_type';
knexJoinFns.push((qry, cpgListId) => qry.leftJoin('campaign_links AS ' + alias, getConds(alias, cpgListId)));
} else {
throw new Error(`Unknown join type "${type}"`);
}
}
const listsFields = {};
const permittedListFields = new Set();
let firstIteration = true;
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const listFields = {};
listsFields[cpgListId] = listFields;
const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId);
const assignedFlds = new Set();
for (const fld of flds) {
/* Dropdown and checkbox groups have field.column == null
For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
if (fld.column) {
const fldKey = 'field:' + fld.key.toLowerCase();
listFields[fldKey] = 'subscriptions.' + fld.column;
if (firstIteration) {
permittedListFields.add(fldKey);
}
if (permittedListFields.has(fldKey)) {
assignedFlds.add(fldKey);
}
}
}
for (const fldKey in [...permittedListFields]) {
if (!assignedFlds.has(fldKey)) {
permittedListFields.delete(fldKey);
}
}
@ -183,36 +287,29 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const listFields = listsFields[cpgListId];
const campaignFieldsMapping = {
'list:id': {raw: knex.raw('?', [cpgListId])},
'tracker:count': {raw: 'COALESCE(`campaign_links`.`count`, 0)'},
'tracker:country': 'campaign_links.country',
'tracker:deviceType': 'campaign_links.device_type',
'tracker:status': 'campaign_messages.status',
'subscription:status': 'subscriptions.status',
'subscription:id': 'subscriptions.id',
'subscription:cid': 'subscriptions.cid',
'subscription:email': 'subscriptions.email'
};
for (const fldKey in listFields) {
if (!permittedListFields.has(fldKey)) {
delete listFields[fldKey];
}
}
}
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const fieldsMapping = {
...commonFieldsMapping,
...campaignFieldsMapping
...listsFields[cpgListId],
'list:id': {raw: knex.raw('?', [cpgListId])}
};
const getColIdIfExists = (colId, getter) => {
if (colId in fieldsMapping) {
return getter(colId);
} else {
throw new Error(`Unknown column id ${colId}`);
}
}
const getSelField = item => {
const itemMapping = fieldsMapping[item];
if (typeof itemMapping === 'string') {
return fieldsMapping[item] + ' AS ' + item;
return itemMapping + ' AS ' + item;
} else if (itemMapping.raw) {
return knex.raw(fieldsMapping[item].raw + ' AS `' + item + '`');
}
@ -230,23 +327,22 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
}
}
let query = knex(`subscription__${cpgListId} AS subscriptions`)
.leftJoin('campaign_messages', {
'campaign_messages.campaign': knex.raw('?', [campaign.id]),
'campaign_messages.list': knex.raw('?', [cpgListId]),
'campaign_messages.subscription': 'subscriptions.id'
})
.leftJoin('campaign_links', {
'campaign_links.campaign': knex.raw('?', [campaign.id]),
'campaign_links.list': knex.raw('?', [cpgListId]),
'campaign_links.subscription': 'subscriptions.id'
})
.select(selFields);
let query = knex(`subscription__${cpgListId} AS subscriptions`).select(selFields);
for (const knexJoinFn of knexJoinFns) {
query = knexJoinFn(query, cpgListId);
}
if (listQryFn) {
query = listQryFn(
query,
colId => getColIdIfExists(colId, x => fieldsMapping[x])
colId => {
if (colId in fieldsMapping) {
return fieldsMapping[colId];
} else {
throw new Error(`Unknown column id ${colId}`);
}
}
);
}
@ -256,13 +352,21 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
if (subsQrys.length > 0) {
let subsSql, subsBindings;
const fieldsSet = new Set([...Object.keys(commonFieldsMapping), ...permittedListFields, 'list:id']);
const applyUnionQryFn = (subsSql, subsBindings) => {
if (unionQryFn) {
return unionQryFn(
knex.from(function() {
return knex.raw('(' + subsSql + ')', subsBindings);
}),
colId => getColIdIfExists(colId, x => x)
colId => {
if (fieldsSet.has(colId)) {
return colId;
} else {
throw new Error(`Unknown column id ${colId}`);
}
}
);
} else {
return knex.raw(subsSql, subsBindings);
@ -311,10 +415,11 @@ async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryF
return await _getCampaignStatistics(
campaign,
select,
[{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}],
unionQryFn,
(qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN)
this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), LinkId.OPEN)
}),
col
),
@ -330,10 +435,11 @@ async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQry
return await _getCampaignStatistics(
campaign,
select,
[{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}],
unionQryFn,
(qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK)
this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), LinkId.GENERAL_CLICK)
}),
col
),
@ -349,10 +455,11 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis
return await _getCampaignStatistics(
campaign,
select,
[{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}],
unionQryFn,
(qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK)
this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), '>', LinkId.GENERAL_CLICK)
}),
col
),
@ -360,12 +467,12 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis
);
}
async function getCampaignStatistics(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, false);
async function getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, false);
}
async function getCampaignStatisticsStream(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, true);
async function getCampaignStatisticsStream(campaign, select, joins, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, true);
}
async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) {
@ -405,6 +512,7 @@ module.exports.remove = remove;
module.exports.updateFields = updateFields;
module.exports.listByState = listByState;
module.exports.bulkChangeState = bulkChangeState;
module.exports.getCampaignCommonListFields = getCampaignCommonListFields;
module.exports.getCampaignStatistics = getCampaignStatistics;
module.exports.getCampaignStatisticsStream = getCampaignStatisticsStream;
module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics;

59
server/routes/quick-reports.js Normal file
View file

@ -0,0 +1,59 @@
'use strict';
const passport = require('../lib/passport');
const shares = require('../models/shares');
const contextHelpers = require('../lib/context-helpers');
const {renderCsvFromStream} = require('../lib/report-helpers');
const reports = require('../models/reports');
const campaigns = require('../models/campaigns');
const {castToInteger} = require('../lib/helpers');
const {SubscriptionStatus} = require('../../shared/lists');
const knex = require('../lib/knex');
const {LinkId} = require('../models/links');
const router = require('../lib/router-async').create();
router.getAsync('/open-and-click-counts/:campaignId', passport.loggedIn, async (req, res) => {
const campaignId = castToInteger(req.params.campaignId);
await shares.enforceEntityPermission(req.context, 'campaign', campaignId, 'viewStats');
const campaign = await campaigns.getById(req.context, campaignId, false);
const listFields = await reports.getCampaignCommonListFields(campaign);
const results = await reports.getCampaignStatisticsStream(
campaign,
['subscription:email', 'open_tracker:count', 'click_tracker:count', 'open_tracker:country', ...Object.keys(listFields)],
[
{type: 'links', prefix: 'open_tracker', onConditions: {link: knex.raw('?', [LinkId.OPEN])} },
{type: 'links', prefix: 'click_tracker', onConditions: {link: knex.raw('?', [LinkId.GENERAL_CLICK])} }
],
null,
(qry, col) => qry
.where(col('subscription:status'), SubscriptionStatus.SUBSCRIBED)
);
res.set({
'Content-Disposition': `attachment;filename=campaign-open-and-click-counts-${campaign.cid}.csv`,
'Content-Type': 'text/csv'
});
await renderCsvFromStream(
results,
res,
{
header: true,
columns: [
{ key: 'subscription:email', header: 'Email' },
{ key: 'open_tracker:count', header: 'Open count' },
{ key: 'click_tracker:count', header: 'Click count' },
{ key: 'open_tracker:country', header: 'Country (first open)' },
...Object.keys(listFields).map(key => ({key, header: listFields[key].key}))
],
delimiter: ','
},
async (row, encoding) => row
);
});
module.exports = router;

6
server/routes/reports.js
View file

@ -5,6 +5,7 @@ const reports = require('../models/reports');
const reportHelpers = require('../lib/report-helpers');
const shares = require('../models/shares');
const contextHelpers = require('../lib/context-helpers');
const {castToInteger} = require('../lib/helpers');
const router = require('../lib/router-async').create();
@ -14,9 +15,10 @@ const fileSuffixes = {
};
router.getAsync('/:id/download', passport.loggedIn, async (req, res) => {
await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
const reportId = castToInteger(req.params.id);
await shares.enforceEntityPermission(req.context, 'report', reportId, 'viewContent');
const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id, false);
const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), reportId, false);
if (report.state == reports.ReportState.FINISHED) {
const headers = {

46
server/services/workers/reports/report-processor.js
View file

@ -6,14 +6,11 @@ const subscriptions = require('../../../models/subscriptions');
const { SubscriptionSource, SubscriptionStatus } = require('../../../../shared/lists');
const campaigns = require('../../../models/campaigns');
const handlebars = require('handlebars');
const hbs = require('hbs');
const vm = require('vm');
const log = require('../../../lib/log');
const fs = require('fs');
const knex = require('../../../lib/knex');
const contextHelpers = require('../../../lib/context-helpers');
const csvStringify = require('csv-stringify');
const {renderCsvFromStream} = require('../../../lib/report-helpers');
const stream = require('stream');
async function main() {
@ -75,46 +72,7 @@ async function main() {
inputs,
SubscriptionSource,
SubscriptionStatus,
renderCsvFromStream: async (readable, opts, transform) => {
const finished = new Promise((success, fail) => {
let lastReadable = readable;
const stringifier = csvStringify(opts);
stringifier.on('finish', () => success());
stringifier.on('error', err => fail(err));
if (transform) {
const rowTransform = new stream.Transform({
objectMode: true,
transform(row, encoding, callback) {
async function performTransform() {
try {
const newRow = await transform(row, encoding);
callback(null, newRow);
} catch (err) {
callback(err);
}
}
// noinspection JSIgnoredPromiseFromCall
performTransform();
}
});
lastReadable.on('error', err => fail(err));
lastReadable.pipe(rowTransform);
lastReadable = rowTransform;
}
stringifier.pipe(process.stdout);
lastReadable.pipe(stringifier);
});
await finished;
},
renderCsvFromStream: (readable, opts, transform) => renderCsvFromStream(readable, process.stdout, opts, transform),
render: data => {
const hbsTmpl = handlebars.compile(report.hbs);