public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fixes in reports. Reports seem to work now

Browse source
This commit is contained in:
Tomas Bures 2018-12-21 19:09:18 +01:00
parent 0be4af5f6c
commit 5a16d789a0
30 changed files with 716 additions and 338 deletions
Download patch file Download diff file Expand all files Collapse all files

6
server/.gitignore vendored
View file

@ -1,7 +1,7 @@
/config/development.*
/config/production.*
/config/test.*
/workers/reports/config/development.*
/workers/reports/config/production.*
/workers/reports/config/test.*
/services/workers/reports/config/development.*
/services/workers/reports/config/production.*
/services/workers/reports/config/test.*
/files

1
server/Gruntfile.js
View file

@ -12,7 +12,6 @@ module.exports = function (grunt) {
'services/**/*.js',
'lib/**/*.js',
'test/**/*.js',
'workers/**/*.js',
'app-builder.js',
'index.js',
'Gruntfile.js',

8
server/app-builder.js
View file

@ -272,13 +272,13 @@ function createApp(appType) {
useWith404Fallback('/codeeditor', sandboxedCodeEditor.getRouter(appType));
if (appType === AppType.TRUSTED || appType === AppType.SANDBOXED) {
if (config.reports && config.reports.enabled === true) {
useWith404Fallback('/reports', reports);
}
useWith404Fallback('/subscriptions', subscriptions);
useWith404Fallback('/webhooks', webhooks);
if (config.reports && config.reports.enabled === true) {
useWith404Fallback('/rpts', reports); // This needs to be different from "reports", which is already used by the UI
}
// API endpoints
useWith404Fallback('/api', api);

6
server/config/default.yaml
View file

@ -217,6 +217,12 @@ testServer:
builtinZoneMTA:
enabled: true
host: localhost
port: 2525
mongo: mongodb://127.0.0.1:27017/zone-mta
redis: redis://localhost:6379/2
log:
level: warn
seleniumWebDriver:
browser: phantomjs

28
server/config/test.toml
View file

@ -1,28 +0,0 @@
[www]
port=3000
[mysql]
user="mailtrain_test"
password="bahquaiphoor"
database="mailtrain_test"
[testServer]
enabled=true
[seleniumWebDriver]
browser="phantomjs"
[ldap]
# enable to use ldap user backend
enabled=false
host="localhost"
port=3002
baseDN="ou=users,dc=example"
filter="(|(username={{username}})(mail={{username}}))"
#Username field in LDAP (uid/cn/username)
uidTag="username"
# nameTag identifies the attribute to be used for user's full name
nameTag="username"
passwordresetlink="xxx"
[reports]
enabled=true
[redis]
enabled=true
[log]
level="verbose"

163
server/lib/builtin-zone-mta.js
View file

@ -4,42 +4,157 @@ const config = require('config');
const fork = require('child_process').fork;
const log = require('./log');
const path = require('path');
const fs = require('fs-extra')
const crypto = require('crypto');
let zoneMtaProcess;
module.exports = {
spawn
};
const zoneMtaDir = path.join(__dirname, '..', '..', 'zone-mta');
const zoneMtaBuiltingConfig = path.join(zoneMtaDir, 'config', 'builtin-zonemta.json')
const password = crypto.randomBytes(20).toString('hex').toLowerCase();
function getUsername() {
return 'mailtrain';
}
function getPassword() {
return password;
}
async function createConfig() {
const cnf = { // This is the main config file
name: 'ZoneMTA',
// Process identifier
ident: 'zone-mta',
// Run as the following user. Only use this if the application starts up as root
user: config.user,
group: config.group,
log: config.builtinZoneMTA.log,
dbs: {
// MongoDB connection string
mongo: config.builtinZoneMTA.mongo,
// Redis connection string
redis: config.builtinZoneMTA.redis,
// Database name for ZoneMTA data in MongoDB. In most cases it should be the same as in the connection string
sender: 'zone-mta'
},
api: {
maildrop: false,
user: getUsername(),
pass: getPassword()
},
smtpInterfaces: {
// Default SMTP interface for accepting mail for delivery
feeder: {
enabled: true,
// How many worker processes to spawn
processes: 1,
// Maximum allowed message size 30MB
maxSize: 31457280,
// Local IP and port to bind to
host: config.builtinZoneMTA.host,
port: config.builtinZoneMTA.port,
// Set to true to require authentication
// If authentication is enabled then you need to use a plugin with an authentication hook
authentication: true,
// How many recipients to allow per message
maxRecipients: 1,
// Set to true to enable STARTTLS. Do not forget to change default TLS keys
starttls: false,
// set to true to start in TLS mode if using port 465
// this probably does not work as TLS support with 465 in ZoneMTA is a bit buggy
secure: false,
}
},
plugins: {
"core/email-bounce": false,
"core/http-bounce": {
enabled: "main",
url: `${config.www.trustedUrlBase}/webhooks/zone-mta`
},
"core/default-headers": {
enabled: ["receiver", "main", "sender"],
futureDate: false,
xOriginatingIP: false
},
'mailtrain-main': {
enabled: ['main']
},
'mailtrain-receiver': {
enabled: ['receiver'],
username: getUsername(),
password: getPassword()
}
},
zones: {
default: {
preferIPv6: false,
ignoreIPv6: true,
processes: 1,
connections: 5,
pool: 'default'
}
}
};
await fs.writeFile(zoneMtaBuiltingConfig, JSON.stringify(cnf, null, 2));
}
function spawn(callback) {
if (config.builtinZoneMTA.enabled) {
log.info('ZoneMTA', 'Starting built-in Zone MTA process');
zoneMtaProcess = fork(
path.join(__dirname, '..', '..', 'zone-mta', 'index.js'),
['--config=' + path.join(__dirname, '..', '..', 'zone-mta', 'config', 'zonemta.js')],
{
cwd: path.join(__dirname, '..', '..', 'zone-mta'),
env: {NODE_ENV: process.env.NODE_ENV}
}
);
createConfig().then(() => {
log.info('ZoneMTA', 'Starting built-in Zone MTA process');
zoneMtaProcess.on('message', msg => {
if (msg) {
if (msg.type === 'zone-mta-started') {
log.info('ZoneMTA', 'ZoneMTA process started');
return callback();
} else if (msg.type === 'entries-added') {
senders.scheduleCheck();
zoneMtaProcess = fork(
path.join(zoneMtaDir, 'index.js'),
['--config=' + zoneMtaBuiltingConfig],
{
cwd: zoneMtaDir,
env: {NODE_ENV: process.env.NODE_ENV}
}
}
});
);
zoneMtaProcess.on('close', (code, signal) => {
log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal);
});
zoneMtaProcess.on('message', msg => {
if (msg) {
if (msg.type === 'zone-mta-started') {
log.info('ZoneMTA', 'ZoneMTA process started');
return callback();
} else if (msg.type === 'entries-added') {
senders.scheduleCheck();
}
}
});
zoneMtaProcess.on('close', (code, signal) => {
log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal);
});
}).catch(err => callback(err));
} else {
callback();
}
}
module.exports.spawn = spawn;
module.exports.getUsername = getUsername;
module.exports.getPassword = getPassword;

3
server/lib/client-helpers.js
View file

@ -47,7 +47,8 @@ async function getAuthenticatedConfig(context) {
mosaico: config.mosaico,
verpEnabled: config.verp.enabled,
reportsEnabled: config.reports.enabled,
mapsApiKey: setts.mapsApiKey
mapsApiKey: setts.mapsApiKey,
builtinZoneMTAEnabled: config.builtinZoneMTA.enabled
}
}

70
server/lib/mailers.js
View file

@ -9,6 +9,8 @@ const nodemailer = require('nodemailer');
const aws = require('aws-sdk');
const openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
const sendConfigurations = require('../models/send-configurations');
const { ZoneMTAType, MailerType } = require('../../shared/send-configurations');
const builtinZoneMta = require('./builtin-zone-mta');
const contextHelpers = require('./context-helpers');
const settings = require('../models/settings');
@ -41,24 +43,28 @@ function invalidateMailer(sendConfigurationId) {
function _addDkimKeys(transport, mail) {
const sendConfiguration = transport.mailer.sendConfiguration;
if (sendConfiguration.mailer_type === sendConfigurations.MailerType.ZONE_MTA) {
if (!mail.headers) {
mail.headers = {};
}
if (sendConfiguration.mailer_type === MailerType.ZONE_MTA) {
const mailerSettings = sendConfiguration.mailer_settings;
const dkimDomain = sendConfiguration.mailer_settings.dkimDomain;
const dkimSelector = (sendConfiguration.mailer_settings.dkimSelector || '').trim();
const dkimPrivateKey = (sendConfiguration.mailer_settings.dkimPrivateKey || '').trim();
if (mailerSettings.zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || mailerSettings.zoneMtaType === ZoneMTAType.BUILTIN) {
if (!mail.headers) {
mail.headers = {};
}
if (dkimSelector && dkimPrivateKey) {
const from = (mail.from.address || '').trim();
const domain = from.split('@').pop().toLowerCase().trim();
const dkimDomain = mailerSettings.dkimDomain;
const dkimSelector = (mailerSettings.dkimSelector || '').trim();
const dkimPrivateKey = (mailerSettings.dkimPrivateKey || '').trim();
mail.headers['x-mailtrain-dkim'] = JSON.stringify({
domainName: dkimDomain || domain,
keySelector: dkimSelector,
privateKey: dkimPrivateKey
});
if (dkimSelector && dkimPrivateKey) {
const from = (mail.from.address || '').trim();
const domain = from.split('@').pop().toLowerCase().trim();
mail.headers['x-mailtrain-dkim'] = JSON.stringify({
domainName: dkimDomain || domain,
keySelector: dkimSelector,
privateKey: dkimPrivateKey
});
}
}
}
}
@ -144,17 +150,9 @@ async function _createTransport(sendConfiguration) {
let transportOptions;
if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
if (mailerType === MailerType.GENERIC_SMTP || mailerType === MailerType.ZONE_MTA) {
transportOptions = {
pool: true,
host: mailerSettings.hostname,
port: mailerSettings.port || false,
secure: mailerSettings.encryption === 'TLS',
ignoreTLS: mailerSettings.encryption === 'NONE',
auth: mailerSettings.useAuth ? {
user: mailerSettings.user,
pass: mailerSettings.password
} : false,
debug: mailerSettings.logTransactions,
logger: mailerSettings.logTransactions ? {
debug: logFunc.bind(null, 'verbose'),
@ -168,7 +166,27 @@ async function _createTransport(sendConfiguration) {
}
};
} else if (mailerType === sendConfigurations.MailerType.AWS_SES) {
if (mailerType === MailerType.ZONE_MTA || mailerSettings.zoneMTAType === ZoneMTAType.BUILTIN) {
transportOptions.host = config.builtinZoneMTA.host;
transportOptions.port = config.builtinZoneMTA.port;
transportOptions.secure = false;
transportOptions.ignoreTLS = true;
transportOptions.auth = {
user: builtinZoneMta.getUsername(),
pass: builtinZoneMta.getPassword()
};
} else {
transportOptions.host = mailerSettings.hostname;
transportOptions.port = mailerSettings.port || false;
transportOptions.secure = mailerSettings.encryption === 'TLS';
transportOptions.ignoreTLS = mailerSettings.encryption === 'NONE';
transportOptions.auth = mailerSettings.useAuth ? {
user: mailerSettings.user,
pass: mailerSettings.password
} : false;
}
} else if (mailerType === MailerType.AWS_SES) {
const sendingRate = mailerSettings.throttling / 3600; // convert to messages/second
transportOptions = {
@ -206,7 +224,7 @@ async function _createTransport(sendConfiguration) {
let throttleWait;
if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
if (mailerType === MailerType.GENERIC_SMTP || mailerType === MailerType.ZONE_MTA) {
let throttling = mailerSettings.throttling;
if (throttling) {
throttling = 1 / (throttling / (3600 * 1000));

262
server/models/reports.js
View file

@ -10,6 +10,10 @@ const namespaceHelpers = require('../lib/namespace-helpers');
const shares = require('./shares');
const reportHelpers = require('../lib/report-helpers');
const fs = require('fs-extra-promise');
const contextHelpers = require('../lib/context-helpers');
const {LinkId} = require('./links');
const subscriptions = require('./subscriptions');
const {Readable} = require('stream');
const ReportState = require('../../shared/reports').ReportState;
@ -118,7 +122,7 @@ async function updateWithConsistencyCheck(context, entity) {
async function removeTx(tx, context, id) {
await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'delete');
const report = tx('reports').where('id', id).first();
const report = await tx('reports').where('id', id).first();
await fs.removeAsync(reportHelpers.getReportContentFile(report));
await fs.removeAsync(reportHelpers.getReportOutputFile(report));
@ -144,55 +148,234 @@ async function bulkChangeState(oldState, newState) {
return await knex('reports').where('state', oldState).update('state', newState);
}
async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, asStream) {
const subsQrys = [];
const campaignFieldsMapping = {
tracker_count: 'campaign_links.count',
country: 'campaign_links.country',
device_type: 'campaign_links.device_type',
status: 'campaign_messages.status',
first_name: 'subscriptions.first_name',
last_name: 'subscriptions.last_name',
email: 'subscriptions.email'
};
const commonFieldsMapping = {};
let firstIteration = true;
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const subsTable = subscriptions.getSubscriptionTableName(cpgListId);
async function getCampaignResults(context, campaign, select, extra) {
const flds = await fields.list(context, campaign.list);
const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId);
const fieldsMapping = Object.assign({}, campaignFieldsMapping);
for (const fld of flds) {
/* Dropdown and checkbox groups have field.column == null
TODO - For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
if (fld.column) {
fieldsMapping[fld.key.toLowerCase()] = 'subscriptions.' + fld.column;
const assignedFlds = new Set();
for (const fld of flds) {
/* Dropdown and checkbox groups have field.column == null
For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
if (fld.column) {
const fldKey = 'field:' + fld.key.toLowerCase();
if (firstIteration || commonFieldsMapping[fldKey]) {
commonFieldsMapping[fldKey] = 'subscriptions.' + fld.column;
assignedFlds.add(fldKey);
}
}
}
for (const fldKey in commonFieldsMapping) {
if (!assignedFlds.has(fldKey)) {
delete commonFieldsMapping[fldKey];
}
}
firstIteration = false;
}
let selFields = [];
for (let idx = 0; idx < select.length; idx++) {
const item = select[idx];
if (item in fieldsMapping) {
selFields.push(fieldsMapping[item] + ' AS ' + item);
} else if (item === '*') {
selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
const subsTable = subscriptions.getSubscriptionTableName(cpgListId);
const campaignFieldsMapping = {
'list:id': {raw: knex.raw('?', [cpgListId])},
'tracker:count': {raw: 'COALESCE(`campaign_links`.`count`, 0)'},
'tracker:country': 'campaign_links.country',
'tracker:deviceType': 'campaign_links.device_type',
'tracker:status': 'campaign_messages.status',
'subscription:status': 'subscriptions.status',
'subscription:id': 'subscriptions.id',
'subscription:cid': 'subscriptions.cid',
'subscription:email': 'subscriptions.email'
};
const fieldsMapping = {
...commonFieldsMapping,
...campaignFieldsMapping
};
const getSelField = item => {
const itemMapping = fieldsMapping[item];
if (typeof itemMapping === 'string') {
return fieldsMapping[item] + ' AS ' + item;
} else if (itemMapping.raw) {
return knex.raw(fieldsMapping[item].raw + ' AS `' + item + '`');
}
};
let selFields = [];
for (let idx = 0; idx < select.length; idx++) {
const item = select[idx];
if (item in fieldsMapping) {
selFields.push(getSelField(item));
} else if (item === '*') {
selFields = selFields.concat(Object.keys(fieldsMapping).map(entry => getSelField(entry)));
} else {
selFields.push(item);
}
}
let query = knex(`subscription__${cpgListId} AS subscriptions`)
.leftJoin('campaign_messages', {
'campaign_messages.subscription': 'subscriptions.id',
'campaign_messages.list': knex.raw('?', [cpgListId])
})
.leftJoin('campaign_links', {
'campaign_links.subscription': 'subscriptions.id',
'campaign_links.list': knex.raw('?', [cpgListId])
})
.select(selFields);
if (listQryFn) {
query = listQryFn(query);
}
subsQrys.push(query.toSQL().toNative());
}
if (subsQrys.length > 0) {
let subsSql, subsBindings;
const applyUnionQryFn = (subsSql, subsBindings) => {
if (unionQryFn) {
return unionQryFn(
knex.from(function() {
return knex.raw('(' + subsSql + ')', subsBindings);
})
);
} else {
return knex.raw(subsSql, subsBindings);
}
}
if (subsQrys.length === 1) {
subsSql = subsQrys[0].sql;
subsBindings = subsQrys[0].bindings;
if (asStream) {
return await applyUnionQryFn(subsSql, subsBindings).stream();
} else {
return await applyUnionQryFn(subsSql, subsBindings);
}
} else {
selFields.push(item);
subsSql = subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ');
subsBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
if (asStream) {
return applyUnionQryFn(subsSql, subsBindings).stream();
} else {
const res = await applyUnionQryFn(subsSql, subsBindings);
if (res[0] && Array.isArray(res[0])) {
return res[0]; // UNION ALL generates an array with result and schema
} else {
return res;
}
}
}
} else {
if (asStream) {
const result = new Readable({
objectMode: true,
});
result.push(null);
return result;
} else {
return [];
}
}
}
let query = knex(`subscription__${campaign.list} AS subscriptions`)
.innerJoin('campaign_messages', 'subscriptions.id', 'campaign_messages.subscription')
.leftJoin('campaign_links', 'subscriptions.id', 'campaign_links.subscription')
.where('campaign_messages.list', campaign.list)
.where('campaign_links.list', campaign.list)
.select(selFields);
if (extra) {
query = extra(query);
async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, asStream) {
if (!listQryFn) {
listQryFn = qry => qry;
}
return await query;
return await _getCampaignStatistics(
campaign,
select,
unionQryFn,
qry => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN)
})
),
asStream
);
}
async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn) {
if (!listQryFn) {
listQryFn = qry => qry;
}
return await _getCampaignStatistics(
campaign,
select,
unionQryFn,
qry => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK)
})
),
asStream
);
}
async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn) {
if (!listQryFn) {
listQryFn = qry => qry;
}
return await _getCampaignStatistics(
campaign,
select,
unionQryFn,
qry => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK)
})
),
asStream
);
}
async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, false);
}
async function getCampaignOpenStatisticsStream(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, true);
}
async function getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn, false);
}
async function getCampaignClickStatisticsStream(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn, true);
}
async function getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn, false);
}
async function getCampaignLinkClickStatisticsStream(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn, true);
}
module.exports.ReportState = ReportState;
@ -205,4 +388,9 @@ module.exports.remove = remove;
module.exports.updateFields = updateFields;
module.exports.listByState = listByState;
module.exports.bulkChangeState = bulkChangeState;
module.exports.getCampaignResults = getCampaignResults;
module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics;
module.exports.getCampaignClickStatistics = getCampaignClickStatistics;
module.exports.getCampaignLinkClickStatistics = getCampaignLinkClickStatistics;
module.exports.getCampaignOpenStatisticsStream = getCampaignOpenStatisticsStream;
module.exports.getCampaignClickStatisticsStream = getCampaignClickStatisticsStream;
module.exports.getCampaignLinkClickStatisticsStream = getCampaignLinkClickStatisticsStream;

1
server/models/send-configurations.js
View file

@ -173,7 +173,6 @@ async function getSystemSendConfiguration() {
return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId(), false);
}
module.exports.MailerType = MailerType;
module.exports.hash = hash;
module.exports.listDTAjax = listDTAjax;
module.exports.listWithSendPermissionDTAjax = listWithSendPermissionDTAjax;

2
server/models/subscriptions.js
View file

@ -369,7 +369,7 @@ async function listTestUsersDTAjax(context, listCid, params) {
});
}
async function list(context, listId, grouped = true, offset, limit) {
async function list(context, listId, grouped, offset, limit) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');

2
server/routes/reports.js
View file

@ -16,7 +16,7 @@ const fileSuffixes = {
router.getAsync('/:id/download', passport.loggedIn, async (req, res) => {
await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id, false);
if (report.state == reports.ReportState.FINISHED) {
const headers = {

7
server/services/executor.js
View file

@ -4,6 +4,7 @@
that can chroot.
*/
const config = require('config');
const reportHelpers = require('../lib/report-helpers');
const fork = require('child_process').fork;
const path = require('path');
@ -111,7 +112,7 @@ process.on('message', msg => {
if (type === 'start-report-processor-worker') {
const ids = privilegeHelpers.getConfigROUidGid();
spawnProcess(msg.tid, path.join(__dirname, '..', 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, '..', 'workers', 'reports'), ids.uid, ids.gid);
spawnProcess(msg.tid, path.join(__dirname, 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, 'workers', 'reports'), ids.uid, ids.gid);
} else if (type === 'stop-process') {
const child = processes[msg.tid];
@ -126,6 +127,10 @@ process.on('message', msg => {
}
});
if (config.title) {
process.title = config.title + ': worker executor';
}
process.send({
type: 'executor-started'
});

5
server/services/feedcheck.js
View file

@ -1,5 +1,6 @@
'use strict';
const config = require('config');
const log = require('../lib/log');
const knex = require('../lib/knex');
const feedparser = require('feedparser-promised');
@ -158,6 +159,10 @@ async function run() {
setTimeout(run, dbCheckInterval);
}
if (config.title) {
process.title = config.title + ': feedcheck';
}
process.send({
type: 'feedcheck-started'
});

5
server/services/importer.js
View file

@ -1,5 +1,6 @@
'use strict';
const config = require('config');
const knex = require('../lib/knex');
const path = require('path');
const log = require('../lib/log');
@ -402,6 +403,10 @@ process.on('message', msg => {
}
});
if (config.title) {
process.title = config.title + ': importer';
}
process.send({
type: 'importer-started'
});

4
server/services/sender-master.js
View file

@ -333,6 +333,10 @@ async function init() {
}
});
if (config.title) {
process.title = config.title + ': sender/master';
}
process.send({
type: 'master-sender-started'
});

4
server/services/sender-worker.js
View file

@ -65,6 +65,10 @@ process.on('message', msg => {
}
});
if (config.title) {
process.title = config.title + ': sender/worker ' + workerId;
}
sendToMaster('worker-started');

13
server/workers/reports/config/default.yaml → server/services/workers/reports/config/default.yaml
View file

@ -1,13 +1,18 @@
# Process title visible in monitoring logs and process listing
title: mailtrain
# Default language to use
language: en
log:
# silly|verbose|info|http|warn|error|silent
level: info
# Default language to use
defaultLanguage: en-US
# Enabled languages
enabledLanguages:
- en-US
- fk-FK
mysql:
host: localhost
user: mailtrain
@ -18,5 +23,5 @@ mysql:
port: 3306
charset: utf8mb4
# The timezone configured on the MySQL server. This can be 'local', 'Z', or an offset in the form +HH:MM or -HH:MM
# If the MySQL server runs on the same server as Mailtrain, use 'local'
timezone: local

51
server/workers/reports/report-processor.js → server/services/workers/reports/report-processor.js
View file

@ -1,34 +1,33 @@
'use strict';
const reports = require('../../models/reports');
const reportTemplates = require('../../models/report-templates');
const lists = require('../../models/lists');
const subscriptions = require('../../models/subscriptions');
const campaigns = require('../../models/campaigns');
const reports = require('../../../models/reports');
const reportTemplates = require('../../../models/report-templates');
const lists = require('../../../models/lists');
const subscriptions = require('../../../models/subscriptions');
const campaigns = require('../../../models/campaigns');
const handlebars = require('handlebars');
const handlebarsHelpers = require('../../lib/handlebars-helpers');
const hbs = require('hbs');
const vm = require('vm');
const log = require('../../lib/log');
const log = require('../../../lib/log');
const fs = require('fs');
const knex = require('../../lib/knex');
const contextHelpers = require('../../lib/context-helpers');
const knex = require('../../../lib/knex');
const contextHelpers = require('../../../lib/context-helpers');
handlebarsHelpers.registerHelpers(handlebars);
const csvStringify = require('csv-stringify');
const stream = require('stream');
async function main() {
try {
const context = contextHelpers.getAdminContext();
const userFieldGetters = {
'campaign': campaigns.getById,
'campaign': id => campaigns.getById(context, id, false, campaigns.Content.ALL),
'list': id => lists.getById(context, id)
};
const reportId = Number(process.argv[2]);
const report = await reports.getByIdWithTemplate(context, reportId);
const report = await reports.getByIdWithTemplate(context, reportId, false);
const inputs = {};
@ -51,22 +50,42 @@ async function main() {
}
const campaignsProxy = {
getResults: (campaign, select, extra) => reports.getCampaignResults(context, campaign, select, extra),
getById: campaignId => campaigns.getById(context, campaignId)
getCampaignOpenStatistics: reports.getCampaignOpenStatistics,
getCampaignClickStatistics: reports.getCampaignClickStatistics,
getCampaignLinkClickStatistics: reports.getCampaignLinkClickStatistics,
getCampaignOpenStatisticsStream: reports.getCampaignOpenStatisticsStream,
getCampaignClickStatisticsStream: reports.getCampaignClickStatisticsStream,
getCampaignLinkClickStatisticsStream: reports.getCampaignLinkClickStatisticsStream,
getById: campaignId => campaigns.getById(context, campaignId, false, campaigns.Content.ALL)
};
const subscriptionsProxy = {
list: listId => subscriptions.list(context, listId)
list: (listId, grouped, offset, limit) => subscriptions.list(context, listId, grouped, offset, limit)
};
const sandbox = {
console,
campaigns: campaignsProxy,
subscriptions: subscriptionsProxy,
stream,
knex,
process,
inputs,
renderCsvFromStream: async (readable, opts) => {
const stringifier = csvStringify(opts);
const finished = new Promise((success, fail) => {
stringifier.on('finish', () => success())
stringifier.on('error', (err) => fail(err))
});
stringifier.pipe(process.stdout);
readable.pipe(stringifier);
await finished;
},
render: data => {
const hbsTmpl = handlebars.compile(report.hbs);
const reportText = hbsTmpl(data);