public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Added quick reports (at this moment only one) to campaign statistics page.

Browse source
This commit is contained in:
Tomas Bures 2019-04-22 22:46:48 +02:00
parent 3e3c3a24fe
commit 72ffe065d2
11 changed files with 305 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

3
server/app-builder.js
View file

@ -21,6 +21,7 @@ const api = require('./routes/api');
// These are routes for the new React-based client
const reports = require('./routes/reports');
const quickReports = require('./routes/quick-reports');
const subscriptions = require('./routes/subscriptions');
const subscription = require('./routes/subscription');
const sandboxedMosaico = require('./routes/sandboxed-mosaico');
@ -286,6 +287,8 @@ async function createApp(appType) {
useWith404Fallback('/rpts', reports); // This needs to be different from "reports", which is already used by the UI
}
useWith404Fallback('/quick-rpts', quickReports);
// API endpoints
useWith404Fallback('/api', api);

45
server/lib/report-helpers.js
View file

@ -1,6 +1,8 @@
'use strict';
const path = require('path');
const csvStringify = require('csv-stringify');
const stream = require('stream');
function nameToFileName(name) {
return name.
@ -25,10 +27,51 @@ function getReportOutputFile(report) {
return getReportFileBase(report) + '.err';
}
async function renderCsvFromStream(readable, writable, opts, transform) {
const finished = new Promise((success, fail) => {
let lastReadable = readable;
const stringifier = csvStringify(opts);
stringifier.on('finish', () => success());
stringifier.on('error', err => fail(err));
if (transform) {
const rowTransform = new stream.Transform({
objectMode: true,
transform(row, encoding, callback) {
async function performTransform() {
try {
const newRow = await transform(row, encoding);
callback(null, newRow);
} catch (err) {
callback(err);
}
}
// noinspection JSIgnoredPromiseFromCall
performTransform();
}
});
lastReadable.on('error', err => fail(err));
lastReadable.pipe(rowTransform);
lastReadable = rowTransform;
}
stringifier.pipe(writable);
lastReadable.pipe(stringifier);
});
await finished;
}
module.exports = {
getReportContentFile,
getReportOutputFile,
nameToFileName,
reportFilesDir
reportFilesDir,
renderCsvFromStream
};

208
server/models/reports.js
View file

@ -148,10 +148,8 @@ async function bulkChangeState(oldState, newState) {
return await knex('reports').where('state', oldState).update('state', newState);
}
async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, asStream) {
const subsQrys = [];
const commonFieldsMapping = {};
async function getCampaignCommonListFields(campaign) {
const listFields = {};
let firstIteration = true;
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
@ -165,16 +163,122 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
if (fld.column) {
const fldKey = 'field:' + fld.key.toLowerCase();
if (firstIteration || commonFieldsMapping[fldKey]) {
commonFieldsMapping[fldKey] = 'subscriptions.' + fld.column;
if (firstIteration) {
listFields[fldKey] = {
key: fld.key,
name: fld.name,
description: fld.description
};
}
if (fldKey in listFields) {
assignedFlds.add(fldKey);
}
}
}
for (const fldKey in commonFieldsMapping) {
for (const fldKey in listFields) {
if (!assignedFlds.has(fldKey)) {
delete commonFieldsMapping[fldKey];
delete listFields[fldKey];
}
}
firstIteration = false;
}
return listFields;
}
async function _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, asStream) {
const subsQrys = [];
joins = joins || [];
const knexJoinFns = [];
const commonFieldsMapping = {
'subscription:status': 'subscriptions.status',
'subscription:id': 'subscriptions.id',
'subscription:cid': 'subscriptions.cid',
'subscription:email': 'subscriptions.email'
};
for (const join of joins) {
const prefix = join.prefix;
const type = join.type;
const onConditions = join.onConditions || {};
const getConds = (alias, cpgListId) => {
const conds = {
[alias + '.campaign']: knex.raw('?', [campaign.id]),
[alias + '.list']: knex.raw('?', [cpgListId]),
[alias + '.subscription']: 'subscriptions.id',
};
for (const onConditionKey in onConditions) {
conds[alias + '.' + onConditionKey] = onConditions[onConditionKey];
}
return conds;
};
if (type === 'messages') {
const alias = 'campaign_messages_' + prefix;
commonFieldsMapping[`${prefix}:status`] = alias + '.status';
knexJoinFns.push((qry, cpgListId) => qry.leftJoin('campaign_messages AS ' + alias, getConds(alias, cpgListId)));
} else if (type === 'links') {
const alias = 'campaign_links_' + prefix;
commonFieldsMapping[`${prefix}:count`] = {raw: 'COALESCE(`' + alias + '`.`count`, 0)'};
commonFieldsMapping[`${prefix}:link`] = alias + '.link';
commonFieldsMapping[`${prefix}:country`] = alias + '.country';
commonFieldsMapping[`${prefix}:deviceType`] = alias + '.device_type';
knexJoinFns.push((qry, cpgListId) => qry.leftJoin('campaign_links AS ' + alias, getConds(alias, cpgListId)));
} else {
throw new Error(`Unknown join type "${type}"`);
}
}
const listsFields = {};
const permittedListFields = new Set();
let firstIteration = true;
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const listFields = {};
listsFields[cpgListId] = listFields;
const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId);
const assignedFlds = new Set();
for (const fld of flds) {
/* Dropdown and checkbox groups have field.column == null
For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
if (fld.column) {
const fldKey = 'field:' + fld.key.toLowerCase();
listFields[fldKey] = 'subscriptions.' + fld.column;
if (firstIteration) {
permittedListFields.add(fldKey);
}
if (permittedListFields.has(fldKey)) {
assignedFlds.add(fldKey);
}
}
}
for (const fldKey in [...permittedListFields]) {
if (!assignedFlds.has(fldKey)) {
permittedListFields.delete(fldKey);
}
}
@ -183,36 +287,29 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const listFields = listsFields[cpgListId];
const campaignFieldsMapping = {
'list:id': {raw: knex.raw('?', [cpgListId])},
'tracker:count': {raw: 'COALESCE(`campaign_links`.`count`, 0)'},
'tracker:country': 'campaign_links.country',
'tracker:deviceType': 'campaign_links.device_type',
'tracker:status': 'campaign_messages.status',
'subscription:status': 'subscriptions.status',
'subscription:id': 'subscriptions.id',
'subscription:cid': 'subscriptions.cid',
'subscription:email': 'subscriptions.email'
};
for (const fldKey in listFields) {
if (!permittedListFields.has(fldKey)) {
delete listFields[fldKey];
}
}
}
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const fieldsMapping = {
...commonFieldsMapping,
...campaignFieldsMapping
...listsFields[cpgListId],
'list:id': {raw: knex.raw('?', [cpgListId])}
};
const getColIdIfExists = (colId, getter) => {
if (colId in fieldsMapping) {
return getter(colId);
} else {
throw new Error(`Unknown column id ${colId}`);
}
}
const getSelField = item => {
const itemMapping = fieldsMapping[item];
if (typeof itemMapping === 'string') {
return fieldsMapping[item] + ' AS ' + item;
return itemMapping + ' AS ' + item;
} else if (itemMapping.raw) {
return knex.raw(fieldsMapping[item].raw + ' AS `' + item + '`');
}
@ -230,23 +327,22 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
}
}
let query = knex(`subscription__${cpgListId} AS subscriptions`)
.leftJoin('campaign_messages', {
'campaign_messages.campaign': knex.raw('?', [campaign.id]),
'campaign_messages.list': knex.raw('?', [cpgListId]),
'campaign_messages.subscription': 'subscriptions.id'
})
.leftJoin('campaign_links', {
'campaign_links.campaign': knex.raw('?', [campaign.id]),
'campaign_links.list': knex.raw('?', [cpgListId]),
'campaign_links.subscription': 'subscriptions.id'
})
.select(selFields);
let query = knex(`subscription__${cpgListId} AS subscriptions`).select(selFields);
for (const knexJoinFn of knexJoinFns) {
query = knexJoinFn(query, cpgListId);
}
if (listQryFn) {
query = listQryFn(
query,
colId => getColIdIfExists(colId, x => fieldsMapping[x])
colId => {
if (colId in fieldsMapping) {
return fieldsMapping[colId];
} else {
throw new Error(`Unknown column id ${colId}`);
}
}
);
}
@ -256,13 +352,21 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
if (subsQrys.length > 0) {
let subsSql, subsBindings;
const fieldsSet = new Set([...Object.keys(commonFieldsMapping), ...permittedListFields, 'list:id']);
const applyUnionQryFn = (subsSql, subsBindings) => {
if (unionQryFn) {
return unionQryFn(
knex.from(function() {
return knex.raw('(' + subsSql + ')', subsBindings);
}),
colId => getColIdIfExists(colId, x => x)
colId => {
if (fieldsSet.has(colId)) {
return colId;
} else {
throw new Error(`Unknown column id ${colId}`);
}
}
);
} else {
return knex.raw(subsSql, subsBindings);
@ -311,10 +415,11 @@ async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryF
return await _getCampaignStatistics(
campaign,
select,
[{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}],
unionQryFn,
(qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN)
this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), LinkId.OPEN)
}),
col
),
@ -330,10 +435,11 @@ async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQry
return await _getCampaignStatistics(
campaign,
select,
[{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}],
unionQryFn,
(qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK)
this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), LinkId.GENERAL_CLICK)
}),
col
),
@ -349,10 +455,11 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis
return await _getCampaignStatistics(
campaign,
select,
[{type: 'messages', prefix: 'tracker'}, {type: 'links', prefix: 'tracker'}],
unionQryFn,
(qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK)
this.whereNull(col('tracker:link')).orWhere(col('tracker:link'), '>', LinkId.GENERAL_CLICK)
}),
col
),
@ -360,12 +467,12 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis
);
}
async function getCampaignStatistics(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, false);
async function getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, false);
}
async function getCampaignStatisticsStream(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, true);
async function getCampaignStatisticsStream(campaign, select, joins, unionQryFn, listQryFn) {
return await _getCampaignStatistics(campaign, select, joins, unionQryFn, listQryFn, true);
}
async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) {
@ -405,6 +512,7 @@ module.exports.remove = remove;
module.exports.updateFields = updateFields;
module.exports.listByState = listByState;
module.exports.bulkChangeState = bulkChangeState;
module.exports.getCampaignCommonListFields = getCampaignCommonListFields;
module.exports.getCampaignStatistics = getCampaignStatistics;
module.exports.getCampaignStatisticsStream = getCampaignStatisticsStream;
module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics;

59
server/routes/quick-reports.js Normal file
View file

@ -0,0 +1,59 @@
'use strict';
const passport = require('../lib/passport');
const shares = require('../models/shares');
const contextHelpers = require('../lib/context-helpers');
const {renderCsvFromStream} = require('../lib/report-helpers');
const reports = require('../models/reports');
const campaigns = require('../models/campaigns');
const {castToInteger} = require('../lib/helpers');
const {SubscriptionStatus} = require('../../shared/lists');
const knex = require('../lib/knex');
const {LinkId} = require('../models/links');
const router = require('../lib/router-async').create();
router.getAsync('/open-and-click-counts/:campaignId', passport.loggedIn, async (req, res) => {
const campaignId = castToInteger(req.params.campaignId);
await shares.enforceEntityPermission(req.context, 'campaign', campaignId, 'viewStats');
const campaign = await campaigns.getById(req.context, campaignId, false);
const listFields = await reports.getCampaignCommonListFields(campaign);
const results = await reports.getCampaignStatisticsStream(
campaign,
['subscription:email', 'open_tracker:count', 'click_tracker:count', 'open_tracker:country', ...Object.keys(listFields)],
[
{type: 'links', prefix: 'open_tracker', onConditions: {link: knex.raw('?', [LinkId.OPEN])} },
{type: 'links', prefix: 'click_tracker', onConditions: {link: knex.raw('?', [LinkId.GENERAL_CLICK])} }
],
null,
(qry, col) => qry
.where(col('subscription:status'), SubscriptionStatus.SUBSCRIBED)
);
res.set({
'Content-Disposition': `attachment;filename=campaign-open-and-click-counts-${campaign.cid}.csv`,
'Content-Type': 'text/csv'
});
await renderCsvFromStream(
results,
res,
{
header: true,
columns: [
{ key: 'subscription:email', header: 'Email' },
{ key: 'open_tracker:count', header: 'Open count' },
{ key: 'click_tracker:count', header: 'Click count' },
{ key: 'open_tracker:country', header: 'Country (first open)' },
...Object.keys(listFields).map(key => ({key, header: listFields[key].key}))
],
delimiter: ','
},
async (row, encoding) => row
);
});
module.exports = router;

6
server/routes/reports.js
View file

@ -5,6 +5,7 @@ const reports = require('../models/reports');
const reportHelpers = require('../lib/report-helpers');
const shares = require('../models/shares');
const contextHelpers = require('../lib/context-helpers');
const {castToInteger} = require('../lib/helpers');
const router = require('../lib/router-async').create();
@ -14,9 +15,10 @@ const fileSuffixes = {
};
router.getAsync('/:id/download', passport.loggedIn, async (req, res) => {
await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
const reportId = castToInteger(req.params.id);
await shares.enforceEntityPermission(req.context, 'report', reportId, 'viewContent');
const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id, false);
const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), reportId, false);
if (report.state == reports.ReportState.FINISHED) {
const headers = {

46
server/services/workers/reports/report-processor.js
View file

@ -6,14 +6,11 @@ const subscriptions = require('../../../models/subscriptions');
const { SubscriptionSource, SubscriptionStatus } = require('../../../../shared/lists');
const campaigns = require('../../../models/campaigns');
const handlebars = require('handlebars');
const hbs = require('hbs');
const vm = require('vm');
const log = require('../../../lib/log');
const fs = require('fs');
const knex = require('../../../lib/knex');
const contextHelpers = require('../../../lib/context-helpers');
const csvStringify = require('csv-stringify');
const {renderCsvFromStream} = require('../../../lib/report-helpers');
const stream = require('stream');
async function main() {
@ -75,46 +72,7 @@ async function main() {
inputs,
SubscriptionSource,
SubscriptionStatus,
renderCsvFromStream: async (readable, opts, transform) => {
const finished = new Promise((success, fail) => {
let lastReadable = readable;
const stringifier = csvStringify(opts);
stringifier.on('finish', () => success());
stringifier.on('error', err => fail(err));
if (transform) {
const rowTransform = new stream.Transform({
objectMode: true,
transform(row, encoding, callback) {
async function performTransform() {
try {
const newRow = await transform(row, encoding);
callback(null, newRow);
} catch (err) {
callback(err);
}
}
// noinspection JSIgnoredPromiseFromCall
performTransform();
}
});
lastReadable.on('error', err => fail(err));
lastReadable.pipe(rowTransform);
lastReadable = rowTransform;
}
stringifier.pipe(process.stdout);
lastReadable.pipe(stringifier);
});
await finished;
},
renderCsvFromStream: (readable, opts, transform) => renderCsvFromStream(readable, process.stdout, opts, transform),
render: data => {
const hbsTmpl = handlebars.compile(report.hbs);