From 5a16d789a057f02643eec53fc434735a47407c65 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Fri, 21 Dec 2018 19:09:18 +0100 Subject: [PATCH] Fixes in reports. Reports seem to work now --- client/src/reports/List.js | 7 +- client/src/reports/Output.js | 57 ---- client/src/reports/View.js | 61 ---- client/src/reports/ViewAndOutput.js | 128 +++++++++ client/src/reports/root.js | 7 +- client/src/reports/templates/CUD.js | 99 ++++--- client/src/reports/templates/List.js | 8 +- client/src/root.js | 2 +- client/src/send-configurations/helpers.js | 40 +-- server/.gitignore | 6 +- server/Gruntfile.js | 1 - server/app-builder.js | 8 +- server/config/default.yaml | 6 + server/config/test.toml | 28 -- server/lib/builtin-zone-mta.js | 163 +++++++++-- server/lib/client-helpers.js | 3 +- server/lib/mailers.js | 70 +++-- server/models/reports.js | 262 +++++++++++++++--- server/models/send-configurations.js | 1 - server/models/subscriptions.js | 2 +- server/routes/reports.js | 2 +- server/services/executor.js | 7 +- server/services/feedcheck.js | 5 + server/services/importer.js | 5 + server/services/sender-master.js | 4 + server/services/sender-worker.js | 4 + .../workers/reports/config/default.yaml | 13 +- .../workers/reports/report-processor.js | 51 ++-- shared/send-configurations.js | 3 +- zone-mta/.gitignore | 1 + 30 files changed, 716 insertions(+), 338 deletions(-) delete mode 100644 client/src/reports/Output.js delete mode 100644 client/src/reports/View.js create mode 100644 client/src/reports/ViewAndOutput.js delete mode 100644 server/config/test.toml rename server/{ => services}/workers/reports/config/default.yaml (79%) rename server/{ => services}/workers/reports/report-processor.js (53%) create mode 100644 zone-mta/.gitignore diff --git a/client/src/reports/List.js b/client/src/reports/List.js index 3e9cd3d6..2d828f55 100644 --- a/client/src/reports/List.js +++ b/client/src/reports/List.js @@ -10,7 +10,10 @@ import axios from '../lib/axios'; import { ReportState } from '../../../shared/reports'; import {Icon} from "../lib/bootstrap-components"; import {checkPermissions} from "../lib/permissions"; -import {getUrl} from "../lib/urls"; +import { + getPublicUrl, + getUrl +} from "../lib/urls"; import { tableAddDeleteButton, tableRestActionDialogInit, @@ -114,7 +117,7 @@ export default class List extends Component { } else if (mimeType === 'text/csv') { viewContent = { label: , - href: `/reports/${id}/download` + href: getUrl(`rpts/${id}/download`) }; } diff --git a/client/src/reports/Output.js b/client/src/reports/Output.js deleted file mode 100644 index bfbcc0a8..00000000 --- a/client/src/reports/Output.js +++ /dev/null @@ -1,57 +0,0 @@ -'use strict'; - -import React, { Component } from 'react'; -import { withTranslation } from '../lib/i18n'; -import { requiresAuthenticatedUser, withPageHelpers, Title } from '../lib/page' -import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling'; -import axios from '../lib/axios'; -import {getUrl} from "../lib/urls"; - -@withTranslation() -@withPageHelpers -@withErrorHandling -@requiresAuthenticatedUser -export default class Output extends Component { - constructor(props) { - super(props); - - this.state = { - output: null - }; - } - - @withAsyncErrorHandler - async loadOutput() { - const id = parseInt(this.props.match.params.reportId); - const outputRespPromise = axios.get(getUrl(`rest/report-output/${id}`)); - const reportRespPromise = axios.get(getUrl(`rest/reports/${id}`)); - const [outputResp, reportResp] = await Promise.all([outputRespPromise, reportRespPromise]); - - this.setState({ - output: outputResp.data, - report: reportResp.data, - }); - } - - componentDidMount() { - // noinspection JSIgnoredPromiseFromCall - this.loadOutput(); - } - - render() { - const t = this.props.t; - - if (this.state.report) { - return ( -
- {t('outputForReportName', { name: this.state.report.name })} - - 
{this.state.output}
-
- ); - } else { - return
{t('loadingReportOutput')}
; - } - - } -} diff --git a/client/src/reports/View.js b/client/src/reports/View.js deleted file mode 100644 index 33fe9c1b..00000000 --- a/client/src/reports/View.js +++ /dev/null @@ -1,61 +0,0 @@ -'use strict'; - -import React, { Component } from 'react'; -import { withTranslation } from '../lib/i18n'; -import { requiresAuthenticatedUser, withPageHelpers, Title } from '../lib/page' -import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling'; -import axios from '../lib/axios'; -import { ReportState } from '../../../shared/reports'; -import {getUrl} from "../lib/urls"; - -@withTranslation() -@withPageHelpers -@withErrorHandling -@requiresAuthenticatedUser -export default class View extends Component { - constructor(props) { - super(props); - - this.state = { - content: null - }; - } - - @withAsyncErrorHandler - async loadContent() { - const id = parseInt(this.props.match.params.reportId); - const contentRespPromise = axios.get(getUrl(`rest/report-content/${id}`)); - const reportRespPromise = axios.get(getUrl(`rest/reports/${id}`)); - const [contentResp, reportResp] = await Promise.all([contentRespPromise, reportRespPromise]); - - this.setState({ - content: contentResp.data, - report: reportResp.data - }); - } - - componentDidMount() { - // noinspection JSIgnoredPromiseFromCall - this.loadContent(); - } - - render() { - const t = this.props.t; - - if (this.state.report) { - if (this.state.report.state === ReportState.FINISHED) { - return ( -
- {t('reportName', { name: this.state.report.name })} - -
-
- ); - } else { - return
{t('reportNotGenerated')}
; - } - } else { - return
{t('loadingReport')}
; - } - } -} diff --git a/client/src/reports/ViewAndOutput.js b/client/src/reports/ViewAndOutput.js new file mode 100644 index 00000000..cec41ca7 --- /dev/null +++ b/client/src/reports/ViewAndOutput.js @@ -0,0 +1,128 @@ +'use strict'; + +import React, { Component } from 'react'; +import { withTranslation } from '../lib/i18n'; +import { + requiresAuthenticatedUser, + withPageHelpers, + Title, + Toolbar, + NavButton +} from '../lib/page' +import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling'; +import axios from '../lib/axios'; +import { ReportState } from '../../../shared/reports'; +import {getUrl} from "../lib/urls"; +import {Button} from "../lib/bootstrap-components"; +import {Link} from "react-router-dom"; +import PropTypes + from "prop-types"; + +@withTranslation() +@withPageHelpers +@withErrorHandling +@requiresAuthenticatedUser +export default class ViewAndOutput extends Component { + constructor(props) { + super(props); + + this.state = { + content: null + }; + + this.reloadTimeoutHandler = ::this.loadContent; + this.reloadTimeoutId = 0; + + const t = props.t; + this.viewTypes = { + view: { + url: 'rest/report-content', + getTitle: name => t('reportName', { name }), + loading: t('loadingReport'), + getContent: content =>
+ }, + output: { + url: 'rest/report-output', + getTitle: name => t('outputForReportName', { name }), + loading: t('loadingReportOutput'), + getContent: content => 
{content}
+ } + } + } + + static propTypes = { + viewType: PropTypes.string.isRequired + } + + @withAsyncErrorHandler + async loadContent() { + const id = parseInt(this.props.match.params.reportId); + const contentRespPromise = axios.get(getUrl(this.viewTypes[this.props.viewType].url + '/' + id)); + const reportRespPromise = axios.get(getUrl(`rest/reports/${id}`)); + const [contentResp, reportResp] = await Promise.all([contentRespPromise, reportRespPromise]); + + this.setState({ + content: contentResp.data, + report: reportResp.data + }); + + const state = reportResp.data.state; + + if (state === ReportState.PROCESSING || state === ReportState.SCHEDULED) { + if (this.reloadTimeoutHandler) { // For some reason the task gets rescheduled if server is restarted while the page is shown. That why we have this check here. + this.reloadTimeoutId = setTimeout(this.reloadTimeoutHandler, 1000); + } + } + } + + componentDidMount() { + // noinspection JSIgnoredPromiseFromCall + this.loadContent(); + } + + componentWillUnmount() { + clearTimeout(this.reloadTimeoutId); + this.reloadTimeoutHandler = null; + } + + + @withAsyncErrorHandler + async refresh() { + const id = parseInt(this.props.match.params.reportId); + await axios.post(getUrl(`rest/report-start/${id}`)); + + // noinspection JSIgnoredPromiseFromCall + this.loadContent(); + } + + render() { + const t = this.props.t; + const viewType = this.viewTypes[this.props.viewType]; + + if (this.state.report) { + let reportContent = null; + + if (this.state.report.state === ReportState.FINISHED) { + reportContent = viewType.getContent(this.state.content); + } else if (this.state.report.state === ReportState.SCHEDULED || this.state.report.state === ReportState.PROCESSING) { + reportContent =
{t('Report is being generated')}
; + } else { + reportContent =
{t('reportNotGenerated')}
; + } + + return ( +
+ +
+ ); + } else { + return
{viewType.loading}
; + } + } +} diff --git a/client/src/reports/root.js b/client/src/reports/root.js index 0a7a65d0..2fd0dff0 100644 --- a/client/src/reports/root.js +++ b/client/src/reports/root.js @@ -3,8 +3,7 @@ import React from 'react'; import ReportsCUD from './CUD'; import ReportsList from './List'; -import ReportsView from './View'; -import ReportsOutput from './Output'; +import ReportsViewAndOutput from './ViewAndOutput'; import ReportTemplatesCUD from './templates/CUD'; import ReportTemplatesList from './templates/List'; import Share from '../shares/Share'; @@ -36,7 +35,7 @@ function getMenus(t) { title: t('view'), link: params => `/reports/${params.reportId}/view`, visible: resolved => resolved.report.permissions.includes('viewContent') && resolved.report.state === ReportState.FINISHED && resolved.report.mime_type === 'text/html', - panelRender: props => (), + panelRender: props => (), }, download: { title: t('download'), @@ -47,7 +46,7 @@ function getMenus(t) { title: t('output'), link: params => `/reports/${params.reportId}/output`, visible: resolved => resolved.report.permissions.includes('viewOutput'), - panelRender: props => () + panelRender: props => () }, share: { title: t('share'), diff --git a/client/src/reports/templates/CUD.js b/client/src/reports/templates/CUD.js index 82684a36..e2d9f7c4 100644 --- a/client/src/reports/templates/CUD.js +++ b/client/src/reports/templates/CUD.js @@ -61,10 +61,10 @@ export default class CUD extends Component { } else { const wizard = this.props.wizard; - if (wizard === 'subscribers-all') { + if (wizard === 'open-counts') { this.populateFormValues({ name: '', - description: 'Generates a campaign report listing all subscribers along with their statistics.', + description: 'Generates a campaign report listing all subscribers along with open counts.', namespace: mailtrainConfig.user.namespace, mime_type: 'text/html', user_fields: @@ -78,13 +78,13 @@ export default class CUD extends Component { ' }\n' + ']', js: - 'const results = await campaigns.getResults(inputs.campaign, ["*"]);\n' + - 'render({ results });', + 'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["*"])\n' + + 'render({ results })', hbs: '

{{title}}

\n' + '\n' + '
\n' + - ' \n' + + '
\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' {{/each}}\n' + @@ -111,10 +111,46 @@ export default class CUD extends Component { '' }); - } else if (wizard === 'subscribers-grouped') { + } else if (wizard === 'open-counts-csv') { this.populateFormValues({ name: '', - description: 'Generates a campaign report with results are aggregated by some "Country" custom field.', + description: 'Generates a campaign report as CSV that lists all subscribers along with open counts.', + namespace: mailtrainConfig.user.namespace, + mime_type: 'text/csv', + user_fields: + '[\n' + + ' {\n' + + ' "id": "campaign",\n' + + ' "name": "Campaign",\n' + + ' "type": "campaign",\n' + + ' "minOccurences": 1,\n' + + ' "maxOccurences": 1\n' + + ' }\n' + + ']', + js: + 'const sampleRowTransform = new stream.Transform({\n' + + ' objectMode: true,\n' + + ' transform(row, encoding, callback) {\n' + + ' callback(null, row)\n' + + ' }\n' + + '})\n' + + '\n' + + 'const results = await campaigns.getCampaignOpenStatisticsStream(inputs.campaign, [\'subscription:email\', \'tracker:count\'])\n' + + '\n' + + 'results.pipe(sampleRowTransform)\n' + + '\n' + + 'await renderCsvFromStream(sampleRowTransform, {\n' + + ' header: true,\n' + + ' columns: [ { key: \'subscription:email\', header: \'Email\' }, { key: \'tracker:count\', header: \'Open count\' } ],\n' + + ' delimiter: \',\'\n' + + '})', + hbs: '' + }); + + } else if (wizard === 'aggregated-open-counts') { + this.populateFormValues({ + name: '', + description: 'Generates a campaign report with results are aggregated by "Country" custom field. (Note that this custom field has to be presents in the subscription custom fields.)', namespace: mailtrainConfig.user.namespace, mime_type: 'text/html', user_fields: @@ -128,22 +164,22 @@ export default class CUD extends Component { ' }\n' + ']', js: - 'const results = await campaigns.getResults(inputs.campaign, ["merge_country"], query =>\n' + + 'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["field:country", "count_opened", "count_all"], query =>\n' + ' query.count("* AS count_all")\n' + - ' .select(knex.raw("SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"))\n' + - ' .groupBy("merge_country")\n' + - ');\n' + + ' .select(knex.raw("SUM(IF(`tracker:count` IS NULL, 0, 1)) AS count_opened"))\n' + + ' .groupBy("field:country")\n' + + ')\n' + '\n' + 'for (const row of results) {\n' + - ' row.percentage = Math.round((row.count_opened / row.count_all) * 100);\n' + + ' row.percentage = Math.round((row["tracker:count"] / row.count_all) * 100)\n' + '}\n' + '\n' + - 'render({ results });', + 'render({ results })', hbs: '

{{title}}

\n' + '\n' + '
\n' + - '
\n' + ' Email\n' + @@ -98,10 +98,10 @@ export default class CUD extends Component { ' {{#each results}}\n' + '
\n' + - ' {{email}}\n' + + ' {{subscription:email}}\n' + ' \n' + - ' {{tracker_count}}\n' + + ' {{tracker:count}}\n' + '
\n' + + '
\n' + ' \n' + ' \n' + ' \n' + '
\n' + ' Country\n' + @@ -163,7 +199,7 @@ export default class CUD extends Component { ' {{#each results}}\n' + '
\n' + - ' {{merge_country}}\n' + + ' {{field:merge_country}}\n' + ' \n' + ' {{count_opened}}\n' + @@ -182,31 +218,6 @@ export default class CUD extends Component { '' }); - } else if (wizard === 'export-list-csv') { - this.populateFormValues({ - name: '', - description: 'Exports a list as a CSV file.', - namespace: mailtrainConfig.user.namespace, - mime_type: 'text/csv', - user_fields: - '[\n' + - ' {\n' + - ' "id": "list",\n' + - ' "name": "List",\n' + - ' "type": "list",\n' + - ' "minOccurences": 1,\n' + - ' "maxOccurences": 1\n' + - ' }\n' + - ']', - js: - 'const results = await subscriptions.list(inputs.list.id);\n' + - 'render({ results });', - hbs: - '{{#each results}}\n' + - '{{firstName}},{{lastName}},{{email}}\n' + - '{{/each}}' - }); - } else { this.populateFormValues({ name: '', @@ -298,7 +309,7 @@ export default class CUD extends Component { - Write the body of the JavaScript function with signature function(inputs, callback) that returns an object to be rendered by the Handlebars template below.}/> + Write the body of the JavaScript function with signature async function(inputs) that returns an object to be rendered by the Handlebars template below.}/> Use HTML with Handlebars syntax. See documentation here.}/> {isEdit ? diff --git a/client/src/reports/templates/List.js b/client/src/reports/templates/List.js index 8626cd28..31029590 100644 --- a/client/src/reports/templates/List.js +++ b/client/src/reports/templates/List.js @@ -74,7 +74,7 @@ export default class List extends Component { }); } - tableAddDeleteButton(actions, this, perms, `rest/reports/templates/${data[0]}`, data[1], t('deletingReportTemplate'), t('reportTemplateDeleted')); + tableAddDeleteButton(actions, this, perms, `rest/report-templates/${data[0]}`, data[1], t('deletingReportTemplate'), t('reportTemplateDeleted')); return actions; } @@ -88,9 +88,9 @@ export default class List extends Component { {t('blank')} - {t('allSubscribers')} - {t('groupedSubscribers')} - {t('exportListAsCsv')} + {t('Open counts')} + {t('Open counts as CSV')} + {t('Aggregrated open counts')} } diff --git a/client/src/root.js b/client/src/root.js index 0d52e6a5..c4105e84 100644 --- a/client/src/root.js +++ b/client/src/root.js @@ -50,7 +50,7 @@ import {getLang} from "../../shared/langs"; const topLevelMenuKeys = ['lists', 'templates', 'campaigns']; -if (mailtrainConfig.reportsEnabmed) { +if (mailtrainConfig.reportsEnabled) { topLevelMenuKeys.push('reports'); } diff --git a/client/src/send-configurations/helpers.js b/client/src/send-configurations/helpers.js index 1ad04dbc..367a97cb 100644 --- a/client/src/send-configurations/helpers.js +++ b/client/src/send-configurations/helpers.js @@ -12,6 +12,7 @@ import { } from "../lib/form"; import {Trans} from "react-i18next"; import styles from "./styles.scss"; +import mailtrainConfig from 'mailtrainConfig'; export const mailerTypesOrder = [ MailerType.ZONE_MTA, @@ -140,11 +141,14 @@ export function getMailerTypes(t) { { key: 'eu-west-1', label: t('euwest1')} ]; - const zoneMtaTypeOptions = [ - { key: ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s Mailtrain plugin (use this option for builtin ZoneMTA)')}, - { key: ZoneMTAType.WITH_HTTP_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s HTTP config plugin')}, - { key: ZoneMTAType.REGULAR, label: t('No dynamic configuration of DKIM keys')} - ] + const zoneMtaTypeOptions = []; + + if (mailtrainConfig.builtinZoneMTAEnabled) { + zoneMtaTypeOptions.push({ key: ZoneMTAType.BUILTIN, label: t('Built-in ZoneMTA')}); + } + zoneMtaTypeOptions.push({ key: ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s Mailtrain plugin (use this option for builtin ZoneMTA)')}); + zoneMtaTypeOptions.push({ key: ZoneMTAType.WITH_HTTP_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s HTTP config plugin')}); + zoneMtaTypeOptions.push({ key: ZoneMTAType.REGULAR, label: t('No dynamic configuration of DKIM keys')}); mailerTypes[MailerType.GENERIC_SMTP] = { getForm: owner => @@ -196,18 +200,22 @@ export function getMailerTypes(t) {
- - - - - { owner.getFormValue('smtpUseAuth') && -
- - -
+ {(zoneMtaType === ZoneMTAType.REGULAR || zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || zoneMtaType === ZoneMTAType.WITH_HTTP_CONF) && +
+ + + + + { owner.getFormValue('smtpUseAuth') && +
+ + +
+ } +
}
- {(zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || zoneMtaType === ZoneMTAType.WITH_HTTP_CONF) && + {(zoneMtaType === ZoneMTAType.BUILTIN || zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || zoneMtaType === ZoneMTAType.WITH_HTTP_CONF) &&

If you are using ZoneMTA then Mailtrain can provide a DKIM key for signing all outgoing messages.

Do not use sensitive keys here. The private key is not encrypted in the database.

 @@ -231,7 +239,7 @@ export function getMailerTypes(t) { }, initData: () => ({ ...getInitGenericSMTP(), - zoneMtaType: ZoneMTAType.REGULAR, + zoneMtaType: mailtrainConfig.builtinZoneMTAEnabled ? ZoneMTAType.BUILTIN : ZoneMTAType.REGULAR, dkimApiKey: '', dkimDomain: '', dkimSelector: '', diff --git a/server/.gitignore b/server/.gitignore index 92ced9a0..70137871 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,7 +1,7 @@ /config/development.* /config/production.* /config/test.* -/workers/reports/config/development.* -/workers/reports/config/production.* -/workers/reports/config/test.* +/services/workers/reports/config/development.* +/services/workers/reports/config/production.* +/services/workers/reports/config/test.* /files diff --git a/server/Gruntfile.js b/server/Gruntfile.js index f9dcbff5..2c915fd2 100644 --- a/server/Gruntfile.js +++ b/server/Gruntfile.js @@ -12,7 +12,6 @@ module.exports = function (grunt) { 'services/**/*.js', 'lib/**/*.js', 'test/**/*.js', - 'workers/**/*.js', 'app-builder.js', 'index.js', 'Gruntfile.js', diff --git a/server/app-builder.js b/server/app-builder.js index b03b904d..a0684ee9 100644 --- a/server/app-builder.js +++ b/server/app-builder.js @@ -272,13 +272,13 @@ function createApp(appType) { useWith404Fallback('/codeeditor', sandboxedCodeEditor.getRouter(appType)); if (appType === AppType.TRUSTED || appType === AppType.SANDBOXED) { - if (config.reports && config.reports.enabled === true) { - useWith404Fallback('/reports', reports); - } - useWith404Fallback('/subscriptions', subscriptions); useWith404Fallback('/webhooks', webhooks); + if (config.reports && config.reports.enabled === true) { + useWith404Fallback('/rpts', reports); // This needs to be different from "reports", which is already used by the UI + } + // API endpoints useWith404Fallback('/api', api); diff --git a/server/config/default.yaml b/server/config/default.yaml index 316ef3df..c454447c 100644 --- a/server/config/default.yaml +++ b/server/config/default.yaml @@ -217,6 +217,12 @@ testServer: builtinZoneMTA: enabled: true + host: localhost + port: 2525 + mongo: mongodb://127.0.0.1:27017/zone-mta + redis: redis://localhost:6379/2 + log: + level: warn seleniumWebDriver: browser: phantomjs diff --git a/server/config/test.toml b/server/config/test.toml deleted file mode 100644 index 106045e0..00000000 --- a/server/config/test.toml +++ /dev/null @@ -1,28 +0,0 @@ -[www] -port=3000 -[mysql] -user="mailtrain_test" -password="bahquaiphoor" -database="mailtrain_test" -[testServer] -enabled=true -[seleniumWebDriver] -browser="phantomjs" -[ldap] -# enable to use ldap user backend -enabled=false -host="localhost" -port=3002 -baseDN="ou=users,dc=example" -filter="(|(username={{username}})(mail={{username}}))" -#Username field in LDAP (uid/cn/username) -uidTag="username" -# nameTag identifies the attribute to be used for user's full name -nameTag="username" -passwordresetlink="xxx" -[reports] -enabled=true -[redis] -enabled=true -[log] -level="verbose" \ No newline at end of file diff --git a/server/lib/builtin-zone-mta.js b/server/lib/builtin-zone-mta.js index d7e6c167..d3014990 100644 --- a/server/lib/builtin-zone-mta.js +++ b/server/lib/builtin-zone-mta.js @@ -4,42 +4,157 @@ const config = require('config'); const fork = require('child_process').fork; const log = require('./log'); const path = require('path'); +const fs = require('fs-extra') +const crypto = require('crypto'); let zoneMtaProcess; -module.exports = { - spawn -}; +const zoneMtaDir = path.join(__dirname, '..', '..', 'zone-mta'); +const zoneMtaBuiltingConfig = path.join(zoneMtaDir, 'config', 'builtin-zonemta.json') + +const password = crypto.randomBytes(20).toString('hex').toLowerCase(); + +function getUsername() { + return 'mailtrain'; +} + +function getPassword() { + return password; +} + +async function createConfig() { + const cnf = { // This is the main config file + name: 'ZoneMTA', + + // Process identifier + ident: 'zone-mta', + + // Run as the following user. Only use this if the application starts up as root + user: config.user, + group: config.group, + + log: config.builtinZoneMTA.log, + + dbs: { + // MongoDB connection string + mongo: config.builtinZoneMTA.mongo, + + // Redis connection string + redis: config.builtinZoneMTA.redis, + + // Database name for ZoneMTA data in MongoDB. In most cases it should be the same as in the connection string + sender: 'zone-mta' + }, + + api: { + maildrop: false, + user: getUsername(), + pass: getPassword() + }, + + smtpInterfaces: { + // Default SMTP interface for accepting mail for delivery + feeder: { + enabled: true, + + // How many worker processes to spawn + processes: 1, + + // Maximum allowed message size 30MB + maxSize: 31457280, + + // Local IP and port to bind to + host: config.builtinZoneMTA.host, + port: config.builtinZoneMTA.port, + + // Set to true to require authentication + // If authentication is enabled then you need to use a plugin with an authentication hook + authentication: true, + + // How many recipients to allow per message + maxRecipients: 1, + + // Set to true to enable STARTTLS. Do not forget to change default TLS keys + starttls: false, + + // set to true to start in TLS mode if using port 465 + // this probably does not work as TLS support with 465 in ZoneMTA is a bit buggy + secure: false, + } + }, + + plugins: { + "core/email-bounce": false, + "core/http-bounce": { + enabled: "main", + url: `${config.www.trustedUrlBase}/webhooks/zone-mta` + }, + "core/default-headers": { + enabled: ["receiver", "main", "sender"], + futureDate: false, + xOriginatingIP: false + }, + 'mailtrain-main': { + enabled: ['main'] + }, + 'mailtrain-receiver': { + enabled: ['receiver'], + username: getUsername(), + password: getPassword() + } + }, + + zones: { + default: { + preferIPv6: false, + ignoreIPv6: true, + processes: 1, + connections: 5, + pool: 'default' + } + } + }; + + await fs.writeFile(zoneMtaBuiltingConfig, JSON.stringify(cnf, null, 2)); +} function spawn(callback) { if (config.builtinZoneMTA.enabled) { - log.info('ZoneMTA', 'Starting built-in Zone MTA process'); - zoneMtaProcess = fork( - path.join(__dirname, '..', '..', 'zone-mta', 'index.js'), - ['--config=' + path.join(__dirname, '..', '..', 'zone-mta', 'config', 'zonemta.js')], - { - cwd: path.join(__dirname, '..', '..', 'zone-mta'), - env: {NODE_ENV: process.env.NODE_ENV} - } - ); + createConfig().then(() => { + log.info('ZoneMTA', 'Starting built-in Zone MTA process'); - zoneMtaProcess.on('message', msg => { - if (msg) { - if (msg.type === 'zone-mta-started') { - log.info('ZoneMTA', 'ZoneMTA process started'); - return callback(); - } else if (msg.type === 'entries-added') { - senders.scheduleCheck(); + zoneMtaProcess = fork( + path.join(zoneMtaDir, 'index.js'), + ['--config=' + zoneMtaBuiltingConfig], + { + cwd: zoneMtaDir, + env: {NODE_ENV: process.env.NODE_ENV} } - } - }); + ); - zoneMtaProcess.on('close', (code, signal) => { - log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal); - }); + zoneMtaProcess.on('message', msg => { + if (msg) { + if (msg.type === 'zone-mta-started') { + log.info('ZoneMTA', 'ZoneMTA process started'); + return callback(); + } else if (msg.type === 'entries-added') { + senders.scheduleCheck(); + } + } + }); + + zoneMtaProcess.on('close', (code, signal) => { + log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal); + }); + + }).catch(err => callback(err)); } else { callback(); } } + +module.exports.spawn = spawn; +module.exports.getUsername = getUsername; +module.exports.getPassword = getPassword; diff --git a/server/lib/client-helpers.js b/server/lib/client-helpers.js index 9d2fc007..ebf0c3ca 100644 --- a/server/lib/client-helpers.js +++ b/server/lib/client-helpers.js @@ -47,7 +47,8 @@ async function getAuthenticatedConfig(context) { mosaico: config.mosaico, verpEnabled: config.verp.enabled, reportsEnabled: config.reports.enabled, - mapsApiKey: setts.mapsApiKey + mapsApiKey: setts.mapsApiKey, + builtinZoneMTAEnabled: config.builtinZoneMTA.enabled } } diff --git a/server/lib/mailers.js b/server/lib/mailers.js index 02449715..8e2f1fec 100644 --- a/server/lib/mailers.js +++ b/server/lib/mailers.js @@ -9,6 +9,8 @@ const nodemailer = require('nodemailer'); const aws = require('aws-sdk'); const openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt; const sendConfigurations = require('../models/send-configurations'); +const { ZoneMTAType, MailerType } = require('../../shared/send-configurations'); +const builtinZoneMta = require('./builtin-zone-mta'); const contextHelpers = require('./context-helpers'); const settings = require('../models/settings'); @@ -41,24 +43,28 @@ function invalidateMailer(sendConfigurationId) { function _addDkimKeys(transport, mail) { const sendConfiguration = transport.mailer.sendConfiguration; - if (sendConfiguration.mailer_type === sendConfigurations.MailerType.ZONE_MTA) { - if (!mail.headers) { - mail.headers = {}; - } + if (sendConfiguration.mailer_type === MailerType.ZONE_MTA) { + const mailerSettings = sendConfiguration.mailer_settings; - const dkimDomain = sendConfiguration.mailer_settings.dkimDomain; - const dkimSelector = (sendConfiguration.mailer_settings.dkimSelector || '').trim(); - const dkimPrivateKey = (sendConfiguration.mailer_settings.dkimPrivateKey || '').trim(); + if (mailerSettings.zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || mailerSettings.zoneMtaType === ZoneMTAType.BUILTIN) { + if (!mail.headers) { + mail.headers = {}; + } - if (dkimSelector && dkimPrivateKey) { - const from = (mail.from.address || '').trim(); - const domain = from.split('@').pop().toLowerCase().trim(); + const dkimDomain = mailerSettings.dkimDomain; + const dkimSelector = (mailerSettings.dkimSelector || '').trim(); + const dkimPrivateKey = (mailerSettings.dkimPrivateKey || '').trim(); - mail.headers['x-mailtrain-dkim'] = JSON.stringify({ - domainName: dkimDomain || domain, - keySelector: dkimSelector, - privateKey: dkimPrivateKey - }); + if (dkimSelector && dkimPrivateKey) { + const from = (mail.from.address || '').trim(); + const domain = from.split('@').pop().toLowerCase().trim(); + + mail.headers['x-mailtrain-dkim'] = JSON.stringify({ + domainName: dkimDomain || domain, + keySelector: dkimSelector, + privateKey: dkimPrivateKey + }); + } } } } @@ -144,17 +150,9 @@ async function _createTransport(sendConfiguration) { let transportOptions; - if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) { + if (mailerType === MailerType.GENERIC_SMTP || mailerType === MailerType.ZONE_MTA) { transportOptions = { pool: true, - host: mailerSettings.hostname, - port: mailerSettings.port || false, - secure: mailerSettings.encryption === 'TLS', - ignoreTLS: mailerSettings.encryption === 'NONE', - auth: mailerSettings.useAuth ? { - user: mailerSettings.user, - pass: mailerSettings.password - } : false, debug: mailerSettings.logTransactions, logger: mailerSettings.logTransactions ? { debug: logFunc.bind(null, 'verbose'), @@ -168,7 +166,27 @@ async function _createTransport(sendConfiguration) { } }; - } else if (mailerType === sendConfigurations.MailerType.AWS_SES) { + if (mailerType === MailerType.ZONE_MTA || mailerSettings.zoneMTAType === ZoneMTAType.BUILTIN) { + transportOptions.host = config.builtinZoneMTA.host; + transportOptions.port = config.builtinZoneMTA.port; + transportOptions.secure = false; + transportOptions.ignoreTLS = true; + transportOptions.auth = { + user: builtinZoneMta.getUsername(), + pass: builtinZoneMta.getPassword() + }; + } else { + transportOptions.host = mailerSettings.hostname; + transportOptions.port = mailerSettings.port || false; + transportOptions.secure = mailerSettings.encryption === 'TLS'; + transportOptions.ignoreTLS = mailerSettings.encryption === 'NONE'; + transportOptions.auth = mailerSettings.useAuth ? { + user: mailerSettings.user, + pass: mailerSettings.password + } : false; + } + + } else if (mailerType === MailerType.AWS_SES) { const sendingRate = mailerSettings.throttling / 3600; // convert to messages/second transportOptions = { @@ -206,7 +224,7 @@ async function _createTransport(sendConfiguration) { let throttleWait; - if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) { + if (mailerType === MailerType.GENERIC_SMTP || mailerType === MailerType.ZONE_MTA) { let throttling = mailerSettings.throttling; if (throttling) { throttling = 1 / (throttling / (3600 * 1000)); diff --git a/server/models/reports.js b/server/models/reports.js index 582f6f03..ca9d02b3 100644 --- a/server/models/reports.js +++ b/server/models/reports.js @@ -10,6 +10,10 @@ const namespaceHelpers = require('../lib/namespace-helpers'); const shares = require('./shares'); const reportHelpers = require('../lib/report-helpers'); const fs = require('fs-extra-promise'); +const contextHelpers = require('../lib/context-helpers'); +const {LinkId} = require('./links'); +const subscriptions = require('./subscriptions'); +const {Readable} = require('stream'); const ReportState = require('../../shared/reports').ReportState; @@ -118,7 +122,7 @@ async function updateWithConsistencyCheck(context, entity) { async function removeTx(tx, context, id) { await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'delete'); - const report = tx('reports').where('id', id).first(); + const report = await tx('reports').where('id', id).first(); await fs.removeAsync(reportHelpers.getReportContentFile(report)); await fs.removeAsync(reportHelpers.getReportOutputFile(report)); @@ -144,55 +148,234 @@ async function bulkChangeState(oldState, newState) { return await knex('reports').where('state', oldState).update('state', newState); } +async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, asStream) { + const subsQrys = []; -const campaignFieldsMapping = { - tracker_count: 'campaign_links.count', - country: 'campaign_links.country', - device_type: 'campaign_links.device_type', - status: 'campaign_messages.status', - first_name: 'subscriptions.first_name', - last_name: 'subscriptions.last_name', - email: 'subscriptions.email' -}; + const commonFieldsMapping = {}; + let firstIteration = true; + for (const cpgList of campaign.lists) { + const cpgListId = cpgList.list; + const subsTable = subscriptions.getSubscriptionTableName(cpgListId); -async function getCampaignResults(context, campaign, select, extra) { - const flds = await fields.list(context, campaign.list); + const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId); - const fieldsMapping = Object.assign({}, campaignFieldsMapping); - for (const fld of flds) { - /* Dropdown and checkbox groups have field.column == null - TODO - For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */ - if (fld.column) { - fieldsMapping[fld.key.toLowerCase()] = 'subscriptions.' + fld.column; + const assignedFlds = new Set(); + + for (const fld of flds) { + /* Dropdown and checkbox groups have field.column == null + For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */ + if (fld.column) { + const fldKey = 'field:' + fld.key.toLowerCase(); + if (firstIteration || commonFieldsMapping[fldKey]) { + commonFieldsMapping[fldKey] = 'subscriptions.' + fld.column; + assignedFlds.add(fldKey); + } + } } + + for (const fldKey in commonFieldsMapping) { + if (!assignedFlds.has(fldKey)) { + delete commonFieldsMapping[fldKey]; + } + } + + firstIteration = false; } - let selFields = []; - for (let idx = 0; idx < select.length; idx++) { - const item = select[idx]; - if (item in fieldsMapping) { - selFields.push(fieldsMapping[item] + ' AS ' + item); - } else if (item === '*') { - selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item)); + for (const cpgList of campaign.lists) { + const cpgListId = cpgList.list; + const subsTable = subscriptions.getSubscriptionTableName(cpgListId); + + const campaignFieldsMapping = { + 'list:id': {raw: knex.raw('?', [cpgListId])}, + 'tracker:count': {raw: 'COALESCE(`campaign_links`.`count`, 0)'}, + 'tracker:country': 'campaign_links.country', + 'tracker:deviceType': 'campaign_links.device_type', + 'tracker:status': 'campaign_messages.status', + 'subscription:status': 'subscriptions.status', + 'subscription:id': 'subscriptions.id', + 'subscription:cid': 'subscriptions.cid', + 'subscription:email': 'subscriptions.email' + }; + + const fieldsMapping = { + ...commonFieldsMapping, + ...campaignFieldsMapping + }; + + const getSelField = item => { + const itemMapping = fieldsMapping[item]; + if (typeof itemMapping === 'string') { + return fieldsMapping[item] + ' AS ' + item; + } else if (itemMapping.raw) { + return knex.raw(fieldsMapping[item].raw + ' AS `' + item + '`'); + } + }; + + let selFields = []; + for (let idx = 0; idx < select.length; idx++) { + const item = select[idx]; + if (item in fieldsMapping) { + selFields.push(getSelField(item)); + } else if (item === '*') { + selFields = selFields.concat(Object.keys(fieldsMapping).map(entry => getSelField(entry))); + } else { + selFields.push(item); + } + } + + let query = knex(`subscription__${cpgListId} AS subscriptions`) + .leftJoin('campaign_messages', { + 'campaign_messages.subscription': 'subscriptions.id', + 'campaign_messages.list': knex.raw('?', [cpgListId]) + }) + .leftJoin('campaign_links', { + 'campaign_links.subscription': 'subscriptions.id', + 'campaign_links.list': knex.raw('?', [cpgListId]) + }) + .select(selFields); + + if (listQryFn) { + query = listQryFn(query); + } + + subsQrys.push(query.toSQL().toNative()); + } + + if (subsQrys.length > 0) { + let subsSql, subsBindings; + + const applyUnionQryFn = (subsSql, subsBindings) => { + if (unionQryFn) { + return unionQryFn( + knex.from(function() { + return knex.raw('(' + subsSql + ')', subsBindings); + }) + ); + } else { + return knex.raw(subsSql, subsBindings); + } + } + + if (subsQrys.length === 1) { + subsSql = subsQrys[0].sql; + subsBindings = subsQrys[0].bindings; + + if (asStream) { + return await applyUnionQryFn(subsSql, subsBindings).stream(); + } else { + return await applyUnionQryFn(subsSql, subsBindings); + } + } else { - selFields.push(item); + subsSql = subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL '); + subsBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings)); + + if (asStream) { + return applyUnionQryFn(subsSql, subsBindings).stream(); + } else { + const res = await applyUnionQryFn(subsSql, subsBindings); + if (res[0] && Array.isArray(res[0])) { + return res[0]; // UNION ALL generates an array with result and schema + } else { + return res; + } + } + } + + } else { + if (asStream) { + const result = new Readable({ + objectMode: true, + }); + result.push(null); + return result; + + } else { + return []; } } +} - let query = knex(`subscription__${campaign.list} AS subscriptions`) - .innerJoin('campaign_messages', 'subscriptions.id', 'campaign_messages.subscription') - .leftJoin('campaign_links', 'subscriptions.id', 'campaign_links.subscription') - .where('campaign_messages.list', campaign.list) - .where('campaign_links.list', campaign.list) - .select(selFields); - - if (extra) { - query = extra(query); +async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, asStream) { + if (!listQryFn) { + listQryFn = qry => qry; } - return await query; + return await _getCampaignStatistics( + campaign, + select, + unionQryFn, + qry => listQryFn( + qry.where(function() { + this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN) + }) + ), + asStream + ); } +async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn) { + if (!listQryFn) { + listQryFn = qry => qry; + } + + return await _getCampaignStatistics( + campaign, + select, + unionQryFn, + qry => listQryFn( + qry.where(function() { + this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK) + }) + ), + asStream + ); +} + +async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn) { + if (!listQryFn) { + listQryFn = qry => qry; + } + + return await _getCampaignStatistics( + campaign, + select, + unionQryFn, + qry => listQryFn( + qry.where(function() { + this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK) + }) + ), + asStream + ); +} + +async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) { + return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, false); +} + +async function getCampaignOpenStatisticsStream(campaign, select, unionQryFn, listQryFn) { + return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, true); +} + +async function getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn) { + return await _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn, false); +} + +async function getCampaignClickStatisticsStream(campaign, select, unionQryFn, listQryFn) { + return await _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn, true); +} + +async function getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn) { + return await _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn, false); +} + +async function getCampaignLinkClickStatisticsStream(campaign, select, unionQryFn, listQryFn) { + return await _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn, true); +} + + module.exports.ReportState = ReportState; @@ -205,4 +388,9 @@ module.exports.remove = remove; module.exports.updateFields = updateFields; module.exports.listByState = listByState; module.exports.bulkChangeState = bulkChangeState; -module.exports.getCampaignResults = getCampaignResults; +module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics; +module.exports.getCampaignClickStatistics = getCampaignClickStatistics; +module.exports.getCampaignLinkClickStatistics = getCampaignLinkClickStatistics; +module.exports.getCampaignOpenStatisticsStream = getCampaignOpenStatisticsStream; +module.exports.getCampaignClickStatisticsStream = getCampaignClickStatisticsStream; +module.exports.getCampaignLinkClickStatisticsStream = getCampaignLinkClickStatisticsStream; diff --git a/server/models/send-configurations.js b/server/models/send-configurations.js index 812a936e..faa3fb11 100644 --- a/server/models/send-configurations.js +++ b/server/models/send-configurations.js @@ -173,7 +173,6 @@ async function getSystemSendConfiguration() { return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId(), false); } -module.exports.MailerType = MailerType; module.exports.hash = hash; module.exports.listDTAjax = listDTAjax; module.exports.listWithSendPermissionDTAjax = listWithSendPermissionDTAjax; diff --git a/server/models/subscriptions.js b/server/models/subscriptions.js index 639b7aae..50cc76ad 100644 --- a/server/models/subscriptions.js +++ b/server/models/subscriptions.js @@ -369,7 +369,7 @@ async function listTestUsersDTAjax(context, listCid, params) { }); } -async function list(context, listId, grouped = true, offset, limit) { +async function list(context, listId, grouped, offset, limit) { return await knex.transaction(async tx => { await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions'); diff --git a/server/routes/reports.js b/server/routes/reports.js index 406b6215..bd1f4d09 100644 --- a/server/routes/reports.js +++ b/server/routes/reports.js @@ -16,7 +16,7 @@ const fileSuffixes = { router.getAsync('/:id/download', passport.loggedIn, async (req, res) => { await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent'); - const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id); + const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id, false); if (report.state == reports.ReportState.FINISHED) { const headers = { diff --git a/server/services/executor.js b/server/services/executor.js index 0c938310..ba08bab2 100644 --- a/server/services/executor.js +++ b/server/services/executor.js @@ -4,6 +4,7 @@ that can chroot. */ +const config = require('config'); const reportHelpers = require('../lib/report-helpers'); const fork = require('child_process').fork; const path = require('path'); @@ -111,7 +112,7 @@ process.on('message', msg => { if (type === 'start-report-processor-worker') { const ids = privilegeHelpers.getConfigROUidGid(); - spawnProcess(msg.tid, path.join(__dirname, '..', 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, '..', 'workers', 'reports'), ids.uid, ids.gid); + spawnProcess(msg.tid, path.join(__dirname, 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, 'workers', 'reports'), ids.uid, ids.gid); } else if (type === 'stop-process') { const child = processes[msg.tid]; @@ -126,6 +127,10 @@ process.on('message', msg => { } }); +if (config.title) { + process.title = config.title + ': worker executor'; +} + process.send({ type: 'executor-started' }); diff --git a/server/services/feedcheck.js b/server/services/feedcheck.js index 7c35599d..537fbb28 100644 --- a/server/services/feedcheck.js +++ b/server/services/feedcheck.js @@ -1,5 +1,6 @@ 'use strict'; +const config = require('config'); const log = require('../lib/log'); const knex = require('../lib/knex'); const feedparser = require('feedparser-promised'); @@ -158,6 +159,10 @@ async function run() { setTimeout(run, dbCheckInterval); } +if (config.title) { + process.title = config.title + ': feedcheck'; +} + process.send({ type: 'feedcheck-started' }); diff --git a/server/services/importer.js b/server/services/importer.js index 78031625..292e01e3 100644 --- a/server/services/importer.js +++ b/server/services/importer.js @@ -1,5 +1,6 @@ 'use strict'; +const config = require('config'); const knex = require('../lib/knex'); const path = require('path'); const log = require('../lib/log'); @@ -402,6 +403,10 @@ process.on('message', msg => { } }); +if (config.title) { + process.title = config.title + ': importer'; +} + process.send({ type: 'importer-started' }); diff --git a/server/services/sender-master.js b/server/services/sender-master.js index 4537ce2e..6ae2f0a4 100644 --- a/server/services/sender-master.js +++ b/server/services/sender-master.js @@ -333,6 +333,10 @@ async function init() { } }); + if (config.title) { + process.title = config.title + ': sender/master'; + } + process.send({ type: 'master-sender-started' }); diff --git a/server/services/sender-worker.js b/server/services/sender-worker.js index 5d250cea..c96cb3a0 100644 --- a/server/services/sender-worker.js +++ b/server/services/sender-worker.js @@ -65,6 +65,10 @@ process.on('message', msg => { } }); +if (config.title) { + process.title = config.title + ': sender/worker ' + workerId; +} + sendToMaster('worker-started'); diff --git a/server/workers/reports/config/default.yaml b/server/services/workers/reports/config/default.yaml similarity index 79% rename from server/workers/reports/config/default.yaml rename to server/services/workers/reports/config/default.yaml index 4b9d7806..3f684680 100644 --- a/server/workers/reports/config/default.yaml +++ b/server/services/workers/reports/config/default.yaml @@ -1,13 +1,18 @@ # Process title visible in monitoring logs and process listing title: mailtrain -# Default language to use -language: en - log: # silly|verbose|info|http|warn|error|silent level: info +# Default language to use +defaultLanguage: en-US + +# Enabled languages +enabledLanguages: +- en-US +- fk-FK + mysql: host: localhost user: mailtrain @@ -18,5 +23,5 @@ mysql: port: 3306 charset: utf8mb4 # The timezone configured on the MySQL server. This can be 'local', 'Z', or an offset in the form +HH:MM or -HH:MM + # If the MySQL server runs on the same server as Mailtrain, use 'local' timezone: local - diff --git a/server/workers/reports/report-processor.js b/server/services/workers/reports/report-processor.js similarity index 53% rename from server/workers/reports/report-processor.js rename to server/services/workers/reports/report-processor.js index 4606a390..9097f2c5 100644 --- a/server/workers/reports/report-processor.js +++ b/server/services/workers/reports/report-processor.js @@ -1,34 +1,33 @@ 'use strict'; -const reports = require('../../models/reports'); -const reportTemplates = require('../../models/report-templates'); -const lists = require('../../models/lists'); -const subscriptions = require('../../models/subscriptions'); -const campaigns = require('../../models/campaigns'); +const reports = require('../../../models/reports'); +const reportTemplates = require('../../../models/report-templates'); +const lists = require('../../../models/lists'); +const subscriptions = require('../../../models/subscriptions'); +const campaigns = require('../../../models/campaigns'); const handlebars = require('handlebars'); -const handlebarsHelpers = require('../../lib/handlebars-helpers'); const hbs = require('hbs'); const vm = require('vm'); -const log = require('../../lib/log'); +const log = require('../../../lib/log'); const fs = require('fs'); -const knex = require('../../lib/knex'); -const contextHelpers = require('../../lib/context-helpers'); +const knex = require('../../../lib/knex'); +const contextHelpers = require('../../../lib/context-helpers'); - -handlebarsHelpers.registerHelpers(handlebars); +const csvStringify = require('csv-stringify'); +const stream = require('stream'); async function main() { try { const context = contextHelpers.getAdminContext(); const userFieldGetters = { - 'campaign': campaigns.getById, + 'campaign': id => campaigns.getById(context, id, false, campaigns.Content.ALL), 'list': id => lists.getById(context, id) }; const reportId = Number(process.argv[2]); - const report = await reports.getByIdWithTemplate(context, reportId); + const report = await reports.getByIdWithTemplate(context, reportId, false); const inputs = {}; @@ -51,22 +50,42 @@ async function main() { } const campaignsProxy = { - getResults: (campaign, select, extra) => reports.getCampaignResults(context, campaign, select, extra), - getById: campaignId => campaigns.getById(context, campaignId) + getCampaignOpenStatistics: reports.getCampaignOpenStatistics, + getCampaignClickStatistics: reports.getCampaignClickStatistics, + getCampaignLinkClickStatistics: reports.getCampaignLinkClickStatistics, + getCampaignOpenStatisticsStream: reports.getCampaignOpenStatisticsStream, + getCampaignClickStatisticsStream: reports.getCampaignClickStatisticsStream, + getCampaignLinkClickStatisticsStream: reports.getCampaignLinkClickStatisticsStream, + getById: campaignId => campaigns.getById(context, campaignId, false, campaigns.Content.ALL) }; const subscriptionsProxy = { - list: listId => subscriptions.list(context, listId) + list: (listId, grouped, offset, limit) => subscriptions.list(context, listId, grouped, offset, limit) }; const sandbox = { console, campaigns: campaignsProxy, subscriptions: subscriptionsProxy, + stream, knex, process, inputs, + renderCsvFromStream: async (readable, opts) => { + const stringifier = csvStringify(opts); + + const finished = new Promise((success, fail) => { + stringifier.on('finish', () => success()) + stringifier.on('error', (err) => fail(err)) + }); + + stringifier.pipe(process.stdout); + readable.pipe(stringifier); + + await finished; + }, + render: data => { const hbsTmpl = handlebars.compile(report.hbs); const reportText = hbsTmpl(data); diff --git a/shared/send-configurations.js b/shared/send-configurations.js index a7b5de27..0091aa63 100644 --- a/shared/send-configurations.js +++ b/shared/send-configurations.js @@ -9,7 +9,8 @@ const MailerType = { const ZoneMTAType = { REGULAR: 0, WITH_HTTP_CONF: 1, - WITH_MAILTRAIN_HEADER_CONF: 2 + WITH_MAILTRAIN_HEADER_CONF: 2, + BUILTIN: 3 } function getSystemSendConfigurationId() { diff --git a/zone-mta/.gitignore b/zone-mta/.gitignore new file mode 100644 index 00000000..b66a28df --- /dev/null +++ b/zone-mta/.gitignore @@ -0,0 +1 @@ +/config/builtin-zonemta.json