Fixes in reports. Reports seem to work now

client/src/reports/List.js

@@ -10,7 +10,10 @@ import axios from '../lib/axios';
 import { ReportState } from '../../../shared/reports';
 import {Icon} from "../lib/bootstrap-components";
 import {checkPermissions} from "../lib/permissions";
-import {getUrl} from "../lib/urls";
+import {
+    getPublicUrl,
+    getUrl
+} from "../lib/urls";
 import {
     tableAddDeleteButton,
     tableRestActionDialogInit,
@@ -114,7 +117,7 @@ export default class List extends Component {
                         } else if (mimeType === 'text/csv') {
                             viewContent = {
                                 label: <Icon icon="download-alt" title={t('download')}/>,
-                                href: `/reports/${id}/download`
+                                href: getUrl(`rpts/${id}/download`)
                             };
                         }
 

client/src/reports/Output.js

@@ -1,57 +0,0 @@
-'use strict';
-
-import React, { Component } from 'react';
-import { withTranslation } from '../lib/i18n';
-import { requiresAuthenticatedUser, withPageHelpers, Title } from '../lib/page'
-import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
-import axios from '../lib/axios';
-import {getUrl} from "../lib/urls";
-
-@withTranslation()
-@withPageHelpers
-@withErrorHandling
-@requiresAuthenticatedUser
-export default class Output extends Component {
-    constructor(props) {
-        super(props);
-
-        this.state = {
-            output: null
-        };
-    }
-
-    @withAsyncErrorHandler
-    async loadOutput() {
-        const id = parseInt(this.props.match.params.reportId);
-        const outputRespPromise = axios.get(getUrl(`rest/report-output/${id}`));
-        const reportRespPromise = axios.get(getUrl(`rest/reports/${id}`));
-        const [outputResp, reportResp] = await Promise.all([outputRespPromise, reportRespPromise]);
-
-        this.setState({
-            output: outputResp.data,
-            report: reportResp.data,
-        });
-    }
-
-    componentDidMount() {
-        // noinspection JSIgnoredPromiseFromCall
-        this.loadOutput();
-    }
-
-    render() {
-        const t = this.props.t;
-
-        if (this.state.report) {
-            return (
-                <div>
-                    <Title>{t('outputForReportName', { name: this.state.report.name })}</Title>
-
-                    <pre>{this.state.output}</pre>
-                </div>
-            );
-        } else {
-            return <div>{t('loadingReportOutput')}</div>;
-        }
-
-    }
-}

client/src/reports/View.js

@@ -1,61 +0,0 @@
-'use strict';
-
-import React, { Component } from 'react';
-import { withTranslation } from '../lib/i18n';
-import { requiresAuthenticatedUser, withPageHelpers, Title } from '../lib/page'
-import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
-import axios from '../lib/axios';
-import { ReportState } from '../../../shared/reports';
-import {getUrl} from "../lib/urls";
-
-@withTranslation()
-@withPageHelpers
-@withErrorHandling
-@requiresAuthenticatedUser
-export default class View extends Component {
-    constructor(props) {
-        super(props);
-
-        this.state = {
-            content: null
-        };
-    }
-
-    @withAsyncErrorHandler
-    async loadContent() {
-        const id = parseInt(this.props.match.params.reportId);
-        const contentRespPromise = axios.get(getUrl(`rest/report-content/${id}`));
-        const reportRespPromise = axios.get(getUrl(`rest/reports/${id}`));
-        const [contentResp, reportResp] = await Promise.all([contentRespPromise, reportRespPromise]);
-
-        this.setState({
-            content: contentResp.data,
-            report: reportResp.data
-        });
-    }
-
-    componentDidMount() {
-        // noinspection JSIgnoredPromiseFromCall
-        this.loadContent();
-    }
-
-    render() {
-        const t = this.props.t;
-
-        if (this.state.report) {
-            if (this.state.report.state === ReportState.FINISHED) {
-                return (
-                    <div>
-                        <Title>{t('reportName', { name: this.state.report.name })}</Title>
-
-                        <div dangerouslySetInnerHTML={{ __html: this.state.content }}/>
-                    </div>
-                );
-            } else {
-                return <div className="alert alert-danger" role="alert">{t('reportNotGenerated')}</div>;
-            }
-        } else {
-            return <div>{t('loadingReport')}</div>;
-        }
-    }
-}

client/src/reports/ViewAndOutput.js

@@ -0,0 +1,128 @@
+'use strict';
+
+import React, { Component } from 'react';
+import { withTranslation } from '../lib/i18n';
+import {
+    requiresAuthenticatedUser,
+    withPageHelpers,
+    Title,
+    Toolbar,
+    NavButton
+} from '../lib/page'
+import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
+import axios from '../lib/axios';
+import { ReportState } from '../../../shared/reports';
+import {getUrl} from "../lib/urls";
+import {Button} from "../lib/bootstrap-components";
+import {Link} from "react-router-dom";
+import PropTypes
+    from "prop-types";
+
+@withTranslation()
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class ViewAndOutput extends Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            content: null
+        };
+
+        this.reloadTimeoutHandler = ::this.loadContent;
+        this.reloadTimeoutId = 0;
+
+        const t = props.t;
+        this.viewTypes = {
+            view: {
+                url: 'rest/report-content',
+                getTitle: name => t('reportName', { name }),
+                loading: t('loadingReport'),
+                getContent: content => <div dangerouslySetInnerHTML={{ __html: content }}/>
+            },
+            output: {
+                url: 'rest/report-output',
+                getTitle: name => t('outputForReportName', { name }),
+                loading: t('loadingReportOutput'),
+                getContent: content => <pre>{content}</pre>
+            }
+        }
+    }
+
+    static propTypes = {
+        viewType: PropTypes.string.isRequired
+    }
+
+    @withAsyncErrorHandler
+    async loadContent() {
+        const id = parseInt(this.props.match.params.reportId);
+        const contentRespPromise = axios.get(getUrl(this.viewTypes[this.props.viewType].url + '/' + id));
+        const reportRespPromise = axios.get(getUrl(`rest/reports/${id}`));
+        const [contentResp, reportResp] = await Promise.all([contentRespPromise, reportRespPromise]);
+
+        this.setState({
+            content: contentResp.data,
+            report: reportResp.data
+        });
+
+        const state = reportResp.data.state;
+
+        if (state === ReportState.PROCESSING || state === ReportState.SCHEDULED) {
+            if (this.reloadTimeoutHandler) { // For some reason the task gets rescheduled if server is restarted while the page is shown. That why we have this check here.
+                this.reloadTimeoutId = setTimeout(this.reloadTimeoutHandler, 1000);
+            }
+        }
+    }
+
+    componentDidMount() {
+        // noinspection JSIgnoredPromiseFromCall
+        this.loadContent();
+    }
+
+    componentWillUnmount() {
+        clearTimeout(this.reloadTimeoutId);
+        this.reloadTimeoutHandler = null;
+    }
+
+
+    @withAsyncErrorHandler
+    async refresh() {
+        const id = parseInt(this.props.match.params.reportId);
+        await axios.post(getUrl(`rest/report-start/${id}`));
+
+        // noinspection JSIgnoredPromiseFromCall
+        this.loadContent();
+    }
+
+    render() {
+        const t = this.props.t;
+        const viewType = this.viewTypes[this.props.viewType];
+
+        if (this.state.report) {
+            let reportContent = null;
+
+            if (this.state.report.state === ReportState.FINISHED) {
+                reportContent = viewType.getContent(this.state.content);
+            } else if (this.state.report.state === ReportState.SCHEDULED || this.state.report.state === ReportState.PROCESSING) {
+                reportContent = <div className="alert alert-info" role="alert">{t('Report is being generated')}</div>;
+            } else {
+                reportContent = <div className="alert alert-danger" role="alert">{t('reportNotGenerated')}</div>;
+            }
+
+            return (
+                <div>
+                    <Toolbar>
+                        <Button className="btn-primary" icon="repeat" label={t('Refresh')} onClickAsync={::this.refresh}/>
+                    </Toolbar>
+
+                    <Title>{viewType.getTitle(this.state.report.name)}</Title>
+
+                    {reportContent}
+                </div>
+            );
+        } else {
+            return <div>{viewType.loading}</div>;
+        }
+    }
+}

client/src/reports/root.js

@@ -3,8 +3,7 @@
 import React from 'react';
 import ReportsCUD from './CUD';
 import ReportsList from './List';
-import ReportsView from './View';
+import ReportsViewAndOutput from './ViewAndOutput';
-import ReportsOutput from './Output';
 import ReportTemplatesCUD from './templates/CUD';
 import ReportTemplatesList from './templates/List';
 import Share from '../shares/Share';
@@ -36,7 +35,7 @@ function getMenus(t) {
                             title: t('view'),
                             link: params => `/reports/${params.reportId}/view`,
                             visible: resolved => resolved.report.permissions.includes('viewContent') && resolved.report.state === ReportState.FINISHED && resolved.report.mime_type === 'text/html',
-                            panelRender: props => (<ReportsView {...props} />),
+                            panelRender: props => (<ReportsViewAndOutput viewType="view" {...props} />),
                         },
                         download: {
                             title: t('download'),
@@ -47,7 +46,7 @@ function getMenus(t) {
                             title: t('output'),
                             link: params => `/reports/${params.reportId}/output`,
                             visible: resolved => resolved.report.permissions.includes('viewOutput'),
-                            panelRender: props => (<ReportsOutput {...props} />)
+                            panelRender: props => (<ReportsViewAndOutput viewType="output" {...props} />)
                         },
                         share: {
                             title: t('share'),

client/src/reports/templates/CUD.js

@@ -61,10 +61,10 @@ export default class CUD extends Component {
         } else {
             const wizard = this.props.wizard;
 
-            if (wizard === 'subscribers-all') {
+            if (wizard === 'open-counts') {
                 this.populateFormValues({
                     name: '',
-                    description: 'Generates a campaign report listing all subscribers along with their statistics.',
+                    description: 'Generates a campaign report listing all subscribers along with open counts.',
                     namespace: mailtrainConfig.user.namespace,
                     mime_type: 'text/html',
                     user_fields:
@@ -78,13 +78,13 @@ export default class CUD extends Component {
                         '  }\n' +
                         ']',
                     js:
-                        'const results = await campaigns.getResults(inputs.campaign, ["*"]);\n' +
+                        'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["*"])\n' +
-                        'render({ results });',
+                        'render({ results })',
                     hbs:
                         '<h2>{{title}}</h2>\n' +
                         '\n' +
                         '<div class="table-responsive">\n' +
-                        '  <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1" data-paging="false">\n' +
+                        '  <table class="table table-bordered table-hover" width="100%">\n' +
                         '    <thead>\n' +
                         '    <th>\n' +
                         '      Email\n' +
@@ -98,10 +98,10 @@ export default class CUD extends Component {
                         '      {{#each results}}\n' +
                         '        <tr>\n' +
                         '          <th scope="row">\n' +
-                        '            {{email}}\n' +
+                        '            {{subscription:email}}\n' +
                         '          </th>\n' +
                         '          <td style="width: 20%;">\n' +
-                        '            {{tracker_count}}\n' +
+                        '            {{tracker:count}}\n' +
                         '          </td>\n' +
                         '        </tr>\n' +
                         '      {{/each}}\n' +
@@ -111,10 +111,46 @@ export default class CUD extends Component {
                         '</div>'
                 });
 
-            } else if (wizard === 'subscribers-grouped') {
+            } else if (wizard === 'open-counts-csv') {
                 this.populateFormValues({
                     name: '',
-                    description: 'Generates a campaign report with results are aggregated by some "Country" custom field.',
+                    description: 'Generates a campaign report as CSV that lists all subscribers along with open counts.',
+                    namespace: mailtrainConfig.user.namespace,
+                    mime_type: 'text/csv',
+                    user_fields:
+                        '[\n' +
+                        '  {\n' +
+                        '    "id": "campaign",\n' +
+                        '    "name": "Campaign",\n' +
+                        '    "type": "campaign",\n' +
+                        '    "minOccurences": 1,\n' +
+                        '    "maxOccurences": 1\n' +
+                        '  }\n' +
+                        ']',
+                    js:
+                        'const sampleRowTransform = new stream.Transform({\n' +
+                        '  objectMode: true,\n' +
+                        '  transform(row, encoding, callback) {\n' +
+                        '    callback(null, row)\n' +
+                        '  }\n' +
+                        '})\n' +
+                        '\n' +
+                        'const results = await campaigns.getCampaignOpenStatisticsStream(inputs.campaign, [\'subscription:email\', \'tracker:count\'])\n' +
+                        '\n' +
+                        'results.pipe(sampleRowTransform)\n' +
+                        '\n' +
+                        'await renderCsvFromStream(sampleRowTransform, {\n' +
+                        '  header: true,\n' +
+                        '  columns: [ { key: \'subscription:email\', header: \'Email\' }, { key: \'tracker:count\', header: \'Open count\' } ],\n' +
+                        '  delimiter: \',\'\n' +
+                        '})',
+                    hbs: ''
+                });
+
+            } else if (wizard === 'aggregated-open-counts') {
+                this.populateFormValues({
+                    name: '',
+                    description: 'Generates a campaign report with results are aggregated by "Country" custom field. (Note that this custom field has to be presents in the subscription custom fields.)',
                     namespace: mailtrainConfig.user.namespace,
                     mime_type: 'text/html',
                     user_fields:
@@ -128,22 +164,22 @@ export default class CUD extends Component {
                         '  }\n' +
                         ']',
                     js:
-                        'const results = await campaigns.getResults(inputs.campaign, ["merge_country"], query =>\n' +
+                        'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["field:country", "count_opened", "count_all"], query =>\n' +
                         '  query.count("* AS count_all")\n' +
-                        '    .select(knex.raw("SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"))\n' +
+                        '    .select(knex.raw("SUM(IF(`tracker:count` IS NULL, 0, 1)) AS count_opened"))\n' +
-                        '    .groupBy("merge_country")\n' +
+                        '    .groupBy("field:country")\n' +
-                        ');\n' +
+                        ')\n' +
                         '\n' +
                         'for (const row of results) {\n' +
-                        '    row.percentage = Math.round((row.count_opened / row.count_all) * 100);\n' +
+                        '    row.percentage = Math.round((row["tracker:count"] / row.count_all) * 100)\n' +
                         '}\n' +
                         '\n' +
-                        'render({ results });',
+                        'render({ results })',
                     hbs:
                         '<h2>{{title}}</h2>\n' +
                         '\n' +
                         '<div class="table-responsive">\n' +
-                        '  <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1,1,1" data-paging="false">\n' +
+                        '  <table class="table table-bordered table-hover" width="100%">\n' +
                         '    <thead>\n' +
                         '      <th>\n' +
                         '        Country\n' +
@@ -163,7 +199,7 @@ export default class CUD extends Component {
                         '    {{#each results}}\n' +
                         '      <tr>\n' +
                         '        <th scope="row">\n' +
-                        '          {{merge_country}}\n' +
+                        '          {{field:merge_country}}\n' +
                         '        </th>\n' +
                         '        <td style="width: 20%;">\n' +
                         '          {{count_opened}}\n' +
@@ -182,31 +218,6 @@ export default class CUD extends Component {
                         '</div>'
                 });
 
-            } else if (wizard === 'export-list-csv') {
-                this.populateFormValues({
-                    name: '',
-                    description: 'Exports a list as a CSV file.',
-                    namespace: mailtrainConfig.user.namespace,
-                    mime_type: 'text/csv',
-                    user_fields:
-                        '[\n' +
-                        '  {\n' +
-                        '    "id": "list",\n' +
-                        '    "name": "List",\n' +
-                        '    "type": "list",\n' +
-                        '    "minOccurences": 1,\n' +
-                        '    "maxOccurences": 1\n' +
-                        '  }\n' +
-                        ']',
-                    js:
-                        'const results = await subscriptions.list(inputs.list.id);\n' +
-                        'render({ results });',
-                    hbs:
-                        '{{#each results}}\n' +
-                        '{{firstName}},{{lastName}},{{email}}\n' +
-                        '{{/each}}'
-                });
-
             } else {
                 this.populateFormValues({
                     name: '',
@@ -298,7 +309,7 @@ export default class CUD extends Component {
                     <DeleteModalDialog
                         stateOwner={this}
                         visible={this.props.action === 'delete'}
-                        deleteUrl={`rest/reports/templates/${this.props.entity.id}`}
+                        deleteUrl={`rest/report-templates/${this.props.entity.id}`}
                         backUrl={`/reports/templates/${this.props.entity.id}/edit`}
                         successUrl="/reports/templates"
                         deletingMsg={t('deletingReportTemplate')}
@@ -313,7 +324,7 @@ export default class CUD extends Component {
                     <Dropdown id="mime_type" label={t('type')} options={[{key: 'text/html', label: t('html')}, {key: 'text/csv', label: t('csv')}]}/>
                     <NamespaceSelect/>
                     <ACEEditor id="user_fields" height="250px" mode="json" label={t('userSelectableFields')} help={t('jsonSpecificationOfUserSelectableFields')}/>
-                    <ACEEditor id="js" height="700px" mode="javascript" label={t('dataProcessingCode')} help={<Trans i18nKey="writeTheBodyOfTheJavaScriptFunctionWith">Write the body of the JavaScript function with signature <code>function(inputs, callback)</code> that returns an object to be rendered by the Handlebars template below.</Trans>}/>
+                    <ACEEditor id="js" height="700px" mode="javascript" label={t('dataProcessingCode')} help={<Trans i18nKey="writeTheBodyOfTheJavaScriptFunctionWith">Write the body of the JavaScript function with signature <code>async function(inputs)</code> that returns an object to be rendered by the Handlebars template below.</Trans>}/>
                     <ACEEditor id="hbs" height="700px" mode="handlebars" label={t('renderingTemplate')} help={<Trans i18nKey="useHtmlWithHandlebarsSyntaxSee">Use HTML with Handlebars syntax. See documentation <a href="http://handlebarsjs.com/">here</a>.</Trans>}/>
 
                     {isEdit ?

client/src/reports/templates/List.js

@@ -74,7 +74,7 @@ export default class List extends Component {
                         });
                     }
 
-                    tableAddDeleteButton(actions, this, perms, `rest/reports/templates/${data[0]}`, data[1], t('deletingReportTemplate'), t('reportTemplateDeleted'));
+                    tableAddDeleteButton(actions, this, perms, `rest/report-templates/${data[0]}`, data[1], t('deletingReportTemplate'), t('reportTemplateDeleted'));
 
                     return actions;
                 }
@@ -88,9 +88,9 @@ export default class List extends Component {
                     <Toolbar>
                         <DropdownMenu className="btn-primary" label={t('createReportTemplate')}>
                             <MenuLink to="/reports/templates/create">{t('blank')}</MenuLink>
-                            <MenuLink to="/reports/templates/create/subscribers-all">{t('allSubscribers')}</MenuLink>
+                            <MenuLink to="/reports/templates/create/open-counts">{t('Open counts')}</MenuLink>
-                            <MenuLink to="/reports/templates/create/subscribers-grouped">{t('groupedSubscribers')}</MenuLink>
+                            <MenuLink to="/reports/templates/create/open-counts-csv">{t('Open counts as CSV')}</MenuLink>
-                            <MenuLink to="/reports/templates/create/export-list-csv">{t('exportListAsCsv')}</MenuLink>
+                            <MenuLink to="/reports/templates/create/aggregated-open-counts">{t('Aggregrated open counts')}</MenuLink>
                         </DropdownMenu>
                     </Toolbar>
                 }

client/src/root.js

@@ -50,7 +50,7 @@ import {getLang} from "../../shared/langs";
 
 const topLevelMenuKeys = ['lists', 'templates', 'campaigns'];
 
-if (mailtrainConfig.reportsEnabmed) {
+if (mailtrainConfig.reportsEnabled) {
     topLevelMenuKeys.push('reports');
 }
 

client/src/send-configurations/helpers.js

@@ -12,6 +12,7 @@ import {
 } from "../lib/form";
 import {Trans} from "react-i18next";
 import styles from "./styles.scss";
+import mailtrainConfig from 'mailtrainConfig';
 
 export const mailerTypesOrder = [
     MailerType.ZONE_MTA,
@@ -140,11 +141,14 @@ export function getMailerTypes(t) {
         { key: 'eu-west-1', label: t('euwest1')}
     ];
 
-    const zoneMtaTypeOptions = [
+    const zoneMtaTypeOptions = [];
-        { key: ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s Mailtrain plugin (use this option for builtin ZoneMTA)')},
+
-        { key: ZoneMTAType.WITH_HTTP_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s HTTP config plugin')},
+    if (mailtrainConfig.builtinZoneMTAEnabled) {
-        { key: ZoneMTAType.REGULAR, label: t('No dynamic configuration of DKIM keys')}
+        zoneMtaTypeOptions.push({ key: ZoneMTAType.BUILTIN, label: t('Built-in ZoneMTA')});
-    ]
+    }
+    zoneMtaTypeOptions.push({ key: ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s Mailtrain plugin (use this option for builtin ZoneMTA)')});
+    zoneMtaTypeOptions.push({ key: ZoneMTAType.WITH_HTTP_CONF, label: t('Dynamic configuration of DKIM keys via ZoneMTA\'s HTTP config plugin')});
+    zoneMtaTypeOptions.push({ key: ZoneMTAType.REGULAR, label: t('No dynamic configuration of DKIM keys')});
 
     mailerTypes[MailerType.GENERIC_SMTP] = {
         getForm: owner =>
@@ -196,6 +200,8 @@ export function getMailerTypes(t) {
                     <Fieldset label={t('mailerSettings')}>
                         <Dropdown id="mailer_type" label={t('mailerType')} options={typeOptions}/>
                         <Dropdown id="zoneMtaType" label={t('Dynamic configuration')} options={zoneMtaTypeOptions}/>
+                        {(zoneMtaType === ZoneMTAType.REGULAR || zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || zoneMtaType === ZoneMTAType.WITH_HTTP_CONF) &&
+                            <div>
                                 <InputField id="smtpHostname" label={t('hostname')} placeholder={t('hostnameEgSmtpexamplecom')}/>
                                 <InputField id="smtpPort" label={t('port')} placeholder={t('portEg465AutodetectedIfLeftBlank')}/>
                                 <Dropdown id="smtpEncryption" label={t('encryption')} options={smtpEncryptionOptions}/>
@@ -206,8 +212,10 @@ export function getMailerTypes(t) {
                                     <InputField id="smtpPassword" label={t('password')} placeholder={t('usernameEgMyaccount@examplecom')}/>
                                 </div>
                                 }
+                            </div>
+                        }
                     </Fieldset>
-                    {(zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || zoneMtaType === ZoneMTAType.WITH_HTTP_CONF) &&
+                    {(zoneMtaType === ZoneMTAType.BUILTIN || zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || zoneMtaType === ZoneMTAType.WITH_HTTP_CONF) &&
                         <Fieldset label={t('dkimSigning')}>
                             <Trans i18nKey="ifYouAreUsingZoneMtaThenMailtrainCan"><p>If you are using ZoneMTA then Mailtrain can provide a DKIM key for signing all outgoing messages.</p></Trans>
                             <Trans i18nKey="doNotUseSensitiveKeysHereThePrivateKeyIs"><p className="text-warning">Do not use sensitive keys here. The private key is not encrypted in the database.</p></Trans>
@@ -231,7 +239,7 @@ export function getMailerTypes(t) {
         },
         initData: () => ({
             ...getInitGenericSMTP(),
-            zoneMtaType: ZoneMTAType.REGULAR,
+            zoneMtaType: mailtrainConfig.builtinZoneMTAEnabled ? ZoneMTAType.BUILTIN : ZoneMTAType.REGULAR,
             dkimApiKey: '',
             dkimDomain: '',
             dkimSelector: '',

server/.gitignore

@@ -1,7 +1,7 @@
 /config/development.*
 /config/production.*
 /config/test.*
-/workers/reports/config/development.*
+/services/workers/reports/config/development.*
-/workers/reports/config/production.*
+/services/workers/reports/config/production.*
-/workers/reports/config/test.*
+/services/workers/reports/config/test.*
 /files

server/Gruntfile.js

@@ -12,7 +12,6 @@ module.exports = function (grunt) {
                 'services/**/*.js',
                 'lib/**/*.js',
                 'test/**/*.js',
-                'workers/**/*.js',
                 'app-builder.js',
                 'index.js',
                 'Gruntfile.js',

server/app-builder.js

@@ -272,13 +272,13 @@ function createApp(appType) {
     useWith404Fallback('/codeeditor', sandboxedCodeEditor.getRouter(appType));
 
     if (appType === AppType.TRUSTED || appType === AppType.SANDBOXED) {
-        if (config.reports && config.reports.enabled === true) {
-            useWith404Fallback('/reports', reports);
-        }
-
         useWith404Fallback('/subscriptions', subscriptions);
         useWith404Fallback('/webhooks', webhooks);
 
+        if (config.reports && config.reports.enabled === true) {
+            useWith404Fallback('/rpts', reports); // This needs to be different from "reports", which is already used by the UI
+        }
+
         // API endpoints
         useWith404Fallback('/api', api);
 

server/config/default.yaml

@@ -217,6 +217,12 @@ testServer:
 
 builtinZoneMTA:
   enabled: true
+  host: localhost
+  port: 2525
+  mongo: mongodb://127.0.0.1:27017/zone-mta
+  redis: redis://localhost:6379/2
+  log:
+    level: warn
 
 seleniumWebDriver:
   browser: phantomjs

server/config/test.toml

@@ -1,28 +0,0 @@
-[www]
-port=3000
-[mysql]
-user="mailtrain_test"
-password="bahquaiphoor"
-database="mailtrain_test"
-[testServer]
-enabled=true
-[seleniumWebDriver]
-browser="phantomjs"
-[ldap]
-# enable to use ldap user backend
-enabled=false
-host="localhost"
-port=3002
-baseDN="ou=users,dc=example"
-filter="(|(username={{username}})(mail={{username}}))"
-#Username field in LDAP (uid/cn/username)
-uidTag="username"
-# nameTag identifies the attribute to be used for user's full name
-nameTag="username"
-passwordresetlink="xxx"
-[reports]
-enabled=true
-[redis]
-enabled=true
-[log]
-level="verbose"

server/lib/builtin-zone-mta.js

@@ -4,22 +4,131 @@ const config = require('config');
 const fork = require('child_process').fork;
 const log = require('./log');
 const path = require('path');
+const fs = require('fs-extra')
+const crypto = require('crypto');
 
 let zoneMtaProcess;
 
-module.exports = {
+const zoneMtaDir = path.join(__dirname, '..', '..', 'zone-mta');
-    spawn
+const zoneMtaBuiltingConfig = path.join(zoneMtaDir, 'config', 'builtin-zonemta.json')
+
+const password = crypto.randomBytes(20).toString('hex').toLowerCase();
+
+function getUsername() {
+    return 'mailtrain';
+}
+
+function getPassword() {
+    return password;
+}
+
+async function createConfig() {
+    const cnf = {    // This is the main config file
+        name: 'ZoneMTA',
+
+        // Process identifier
+        ident: 'zone-mta',
+
+        // Run as the following user. Only use this if the application starts up as root
+        user: config.user,
+        group: config.group,
+
+        log: config.builtinZoneMTA.log,
+
+        dbs: {
+            // MongoDB connection string
+            mongo: config.builtinZoneMTA.mongo,
+
+            // Redis connection string
+            redis: config.builtinZoneMTA.redis,
+
+            // Database name for ZoneMTA data in MongoDB. In most cases it should be the same as in the connection string
+            sender: 'zone-mta'
+        },
+
+        api: {
+            maildrop: false,
+            user: getUsername(),
+            pass: getPassword()
+        },
+
+        smtpInterfaces: {
+            // Default SMTP interface for accepting mail for delivery
+            feeder: {
+                enabled: true,
+
+                // How many worker processes to spawn
+                processes: 1,
+
+                // Maximum allowed message size 30MB
+                maxSize: 31457280,
+
+                // Local IP and port to bind to
+                host: config.builtinZoneMTA.host,
+                port: config.builtinZoneMTA.port,
+
+                // Set to true to require authentication
+                // If authentication is enabled then you need to use a plugin with an authentication hook
+                authentication: true,
+
+                // How many recipients to allow per message
+                maxRecipients: 1,
+
+                // Set to true to enable STARTTLS. Do not forget to change default TLS keys
+                starttls: false,
+
+                // set to true to start in TLS mode if using port 465
+                // this probably does not work as TLS support with 465 in ZoneMTA is a bit buggy
+                secure: false,
+            }
+        },
+
+        plugins: {
+            "core/email-bounce": false,
+            "core/http-bounce": {
+                enabled: "main",
+                url: `${config.www.trustedUrlBase}/webhooks/zone-mta`
+            },
+            "core/default-headers": {
+                enabled: ["receiver", "main", "sender"],
+                futureDate: false,
+                xOriginatingIP: false
+            },
+            'mailtrain-main': {
+                enabled: ['main']
+            },
+            'mailtrain-receiver': {
+                enabled: ['receiver'],
+                username: getUsername(),
+                password: getPassword()
+            }
+        },
+
+        zones: {
+            default: {
+                preferIPv6: false,
+                ignoreIPv6: true,
+                processes: 1,
+                connections: 5,
+                pool: 'default'
+            }
+        }
     };
 
+    await fs.writeFile(zoneMtaBuiltingConfig, JSON.stringify(cnf, null, 2));
+}
+
 function spawn(callback) {
     if (config.builtinZoneMTA.enabled) {
+
+        createConfig().then(() => {
             log.info('ZoneMTA', 'Starting built-in Zone MTA process');
 
             zoneMtaProcess = fork(
-            path.join(__dirname, '..', '..', 'zone-mta', 'index.js'),
+                path.join(zoneMtaDir, 'index.js'),
-            ['--config=' + path.join(__dirname, '..', '..', 'zone-mta', 'config', 'zonemta.js')],
+                ['--config=' + zoneMtaBuiltingConfig],
                 {
-                cwd: path.join(__dirname, '..', '..', 'zone-mta'),
+                    cwd: zoneMtaDir,
                     env: {NODE_ENV: process.env.NODE_ENV}
                 }
             );
@@ -39,7 +148,13 @@ function spawn(callback) {
                 log.error('ZoneMTA', 'ZoneMTA process exited with code %s signal %s', code, signal);
             });
 
+        }).catch(err => callback(err));
+
     } else {
         callback();
     }
 }
+
+module.exports.spawn = spawn;
+module.exports.getUsername = getUsername;
+module.exports.getPassword = getPassword;

server/lib/client-helpers.js

@@ -47,7 +47,8 @@ async function getAuthenticatedConfig(context) {
         mosaico: config.mosaico,
         verpEnabled: config.verp.enabled,
         reportsEnabled: config.reports.enabled,
-        mapsApiKey: setts.mapsApiKey
+        mapsApiKey: setts.mapsApiKey,
+        builtinZoneMTAEnabled: config.builtinZoneMTA.enabled
     }
 }
 

server/lib/mailers.js

@@ -9,6 +9,8 @@ const nodemailer = require('nodemailer');
 const aws = require('aws-sdk');
 const openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
 const sendConfigurations = require('../models/send-configurations');
+const { ZoneMTAType, MailerType } = require('../../shared/send-configurations');
+const builtinZoneMta = require('./builtin-zone-mta');
 
 const contextHelpers = require('./context-helpers');
 const settings = require('../models/settings');
@@ -41,14 +43,17 @@ function invalidateMailer(sendConfigurationId) {
 function _addDkimKeys(transport, mail) {
     const sendConfiguration = transport.mailer.sendConfiguration;
 
-    if (sendConfiguration.mailer_type === sendConfigurations.MailerType.ZONE_MTA) {
+    if (sendConfiguration.mailer_type === MailerType.ZONE_MTA) {
+        const mailerSettings = sendConfiguration.mailer_settings;
+
+        if (mailerSettings.zoneMtaType === ZoneMTAType.WITH_MAILTRAIN_HEADER_CONF || mailerSettings.zoneMtaType === ZoneMTAType.BUILTIN) {
             if (!mail.headers) {
                 mail.headers = {};
             }
 
-        const dkimDomain = sendConfiguration.mailer_settings.dkimDomain;
+            const dkimDomain = mailerSettings.dkimDomain;
-        const dkimSelector = (sendConfiguration.mailer_settings.dkimSelector || '').trim();
+            const dkimSelector = (mailerSettings.dkimSelector || '').trim();
-        const dkimPrivateKey = (sendConfiguration.mailer_settings.dkimPrivateKey || '').trim();
+            const dkimPrivateKey = (mailerSettings.dkimPrivateKey || '').trim();
 
             if (dkimSelector && dkimPrivateKey) {
                 const from = (mail.from.address || '').trim();
@@ -62,6 +67,7 @@ function _addDkimKeys(transport, mail) {
             }
         }
     }
+}
 
 
 async function _sendMail(transport, mail, template) {
@@ -144,17 +150,9 @@ async function _createTransport(sendConfiguration) {
 
     let transportOptions;
 
-    if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
+    if (mailerType === MailerType.GENERIC_SMTP || mailerType === MailerType.ZONE_MTA) {
         transportOptions = {
             pool: true,
-            host: mailerSettings.hostname,
-            port: mailerSettings.port || false,
-            secure: mailerSettings.encryption === 'TLS',
-            ignoreTLS: mailerSettings.encryption === 'NONE',
-            auth: mailerSettings.useAuth ? {
-                user: mailerSettings.user,
-                pass: mailerSettings.password
-            } : false,
             debug: mailerSettings.logTransactions,
             logger: mailerSettings.logTransactions ? {
                 debug: logFunc.bind(null, 'verbose'),
@@ -168,7 +166,27 @@ async function _createTransport(sendConfiguration) {
             }
         };
 
-    } else if (mailerType === sendConfigurations.MailerType.AWS_SES) {
+        if (mailerType === MailerType.ZONE_MTA || mailerSettings.zoneMTAType === ZoneMTAType.BUILTIN) {
+            transportOptions.host = config.builtinZoneMTA.host;
+            transportOptions.port = config.builtinZoneMTA.port;
+            transportOptions.secure = false;
+            transportOptions.ignoreTLS = true;
+            transportOptions.auth = {
+                user: builtinZoneMta.getUsername(),
+                pass: builtinZoneMta.getPassword()
+            };
+        } else {
+            transportOptions.host = mailerSettings.hostname;
+            transportOptions.port = mailerSettings.port || false;
+            transportOptions.secure = mailerSettings.encryption === 'TLS';
+            transportOptions.ignoreTLS = mailerSettings.encryption === 'NONE';
+            transportOptions.auth = mailerSettings.useAuth ? {
+                user: mailerSettings.user,
+                pass: mailerSettings.password
+            } : false;
+        }
+
+    } else if (mailerType === MailerType.AWS_SES) {
         const sendingRate = mailerSettings.throttling / 3600;  // convert to messages/second
 
         transportOptions = {
@@ -206,7 +224,7 @@ async function _createTransport(sendConfiguration) {
 
     let throttleWait;
 
-    if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
+    if (mailerType === MailerType.GENERIC_SMTP || mailerType === MailerType.ZONE_MTA) {
         let throttling = mailerSettings.throttling;
         if (throttling) {
             throttling = 1 / (throttling / (3600 * 1000));

server/models/reports.js

@@ -10,6 +10,10 @@ const namespaceHelpers = require('../lib/namespace-helpers');
 const shares = require('./shares');
 const reportHelpers = require('../lib/report-helpers');
 const fs = require('fs-extra-promise');
+const contextHelpers = require('../lib/context-helpers');
+const {LinkId} = require('./links');
+const subscriptions = require('./subscriptions');
+const {Readable} = require('stream');
 
 const ReportState = require('../../shared/reports').ReportState;
 
@@ -118,7 +122,7 @@ async function updateWithConsistencyCheck(context, entity) {
 async function removeTx(tx, context, id) {
     await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'delete');
 
-    const report = tx('reports').where('id', id).first();
+    const report = await tx('reports').where('id', id).first();
 
     await fs.removeAsync(reportHelpers.getReportContentFile(report));
     await fs.removeAsync(reportHelpers.getReportOutputFile(report));
@@ -144,55 +148,234 @@ async function bulkChangeState(oldState, newState) {
     return await knex('reports').where('state', oldState).update('state', newState);
 }
 
+async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, asStream) {
+    const subsQrys = [];
 
-const campaignFieldsMapping = {
+    const commonFieldsMapping = {};
-    tracker_count: 'campaign_links.count',
+    let firstIteration = true;
-    country: 'campaign_links.country',
+    for (const cpgList of campaign.lists) {
-    device_type: 'campaign_links.device_type',
+        const cpgListId = cpgList.list;
-    status: 'campaign_messages.status',
+        const subsTable = subscriptions.getSubscriptionTableName(cpgListId);
-    first_name: 'subscriptions.first_name',
-    last_name: 'subscriptions.last_name',
-    email: 'subscriptions.email'
-};
 
-async function getCampaignResults(context, campaign, select, extra) {
+        const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId);
-    const flds = await fields.list(context, campaign.list);
+
+        const assignedFlds = new Set();
 
-    const fieldsMapping = Object.assign({}, campaignFieldsMapping);
         for (const fld of flds) {
             /* Dropdown and checkbox groups have field.column == null
-           TODO - For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
+               For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
             if (fld.column) {
-            fieldsMapping[fld.key.toLowerCase()] = 'subscriptions.' + fld.column;
+                const fldKey = 'field:' + fld.key.toLowerCase();
+                if (firstIteration || commonFieldsMapping[fldKey]) {
+                    commonFieldsMapping[fldKey] = 'subscriptions.' + fld.column;
+                    assignedFlds.add(fldKey);
                 }
             }
+        }
+
+        for (const fldKey in commonFieldsMapping) {
+            if (!assignedFlds.has(fldKey)) {
+                delete commonFieldsMapping[fldKey];
+            }
+        }
+
+        firstIteration = false;
+    }
+
+    for (const cpgList of campaign.lists) {
+        const cpgListId = cpgList.list;
+        const subsTable = subscriptions.getSubscriptionTableName(cpgListId);
+
+        const campaignFieldsMapping = {
+            'list:id': {raw: knex.raw('?', [cpgListId])},
+            'tracker:count': {raw: 'COALESCE(`campaign_links`.`count`, 0)'},
+            'tracker:country': 'campaign_links.country',
+            'tracker:deviceType': 'campaign_links.device_type',
+            'tracker:status': 'campaign_messages.status',
+            'subscription:status': 'subscriptions.status',
+            'subscription:id': 'subscriptions.id',
+            'subscription:cid': 'subscriptions.cid',
+            'subscription:email': 'subscriptions.email'
+        };
+
+        const fieldsMapping = {
+            ...commonFieldsMapping,
+            ...campaignFieldsMapping
+        };
+
+        const getSelField = item => {
+            const itemMapping = fieldsMapping[item];
+            if (typeof itemMapping === 'string') {
+                return fieldsMapping[item] + ' AS ' + item;
+            } else if (itemMapping.raw) {
+                return knex.raw(fieldsMapping[item].raw + ' AS `' + item + '`');
+            }
+        };
 
         let selFields = [];
         for (let idx = 0; idx < select.length; idx++) {
             const item = select[idx];
             if (item in fieldsMapping) {
-            selFields.push(fieldsMapping[item] + ' AS ' + item);
+                selFields.push(getSelField(item));
             } else if (item === '*') {
-            selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
+                selFields = selFields.concat(Object.keys(fieldsMapping).map(entry => getSelField(entry)));
             } else {
                 selFields.push(item);
             }
         }
 
-    let query = knex(`subscription__${campaign.list} AS subscriptions`)
+        let query = knex(`subscription__${cpgListId} AS subscriptions`)
-        .innerJoin('campaign_messages', 'subscriptions.id', 'campaign_messages.subscription')
+            .leftJoin('campaign_messages', {
-        .leftJoin('campaign_links', 'subscriptions.id', 'campaign_links.subscription')
+                'campaign_messages.subscription': 'subscriptions.id',
-        .where('campaign_messages.list', campaign.list)
+                'campaign_messages.list': knex.raw('?', [cpgListId])
-        .where('campaign_links.list', campaign.list)
+            })
+            .leftJoin('campaign_links', {
+                'campaign_links.subscription': 'subscriptions.id',
+                'campaign_links.list': knex.raw('?', [cpgListId])
+            })
             .select(selFields);
 
-    if (extra) {
+        if (listQryFn) {
-        query = extra(query);
+            query = listQryFn(query);
         }
 
-    return await query;
+        subsQrys.push(query.toSQL().toNative());
     }
 
+    if (subsQrys.length > 0) {
+        let subsSql, subsBindings;
+
+        const applyUnionQryFn = (subsSql, subsBindings) => {
+            if (unionQryFn) {
+                return unionQryFn(
+                    knex.from(function() {
+                        return knex.raw('(' + subsSql + ')', subsBindings);
+                    })
+                );
+            } else {
+                return knex.raw(subsSql, subsBindings);
+            }
+        }
+
+        if (subsQrys.length === 1) {
+            subsSql = subsQrys[0].sql;
+            subsBindings = subsQrys[0].bindings;
+
+            if (asStream) {
+                return await applyUnionQryFn(subsSql, subsBindings).stream();
+            } else {
+                return await applyUnionQryFn(subsSql, subsBindings);
+            }
+
+        } else {
+            subsSql = subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ');
+            subsBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+
+            if (asStream) {
+                return applyUnionQryFn(subsSql, subsBindings).stream();
+            } else {
+                const res = await applyUnionQryFn(subsSql, subsBindings);
+                if (res[0] && Array.isArray(res[0])) {
+                    return res[0]; // UNION ALL generates an array with result and schema
+                } else {
+                    return res;
+                }
+            }
+        }
+
+    } else {
+        if (asStream) {
+            const result = new Readable({
+                objectMode: true,
+            });
+            result.push(null);
+            return result;
+            
+        } else {
+            return [];
+        }
+    }
+}
+
+async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, asStream) {
+    if (!listQryFn) {
+        listQryFn = qry => qry;
+    }
+
+    return await _getCampaignStatistics(
+        campaign,
+        select,
+        unionQryFn,
+        qry => listQryFn(
+            qry.where(function() {
+                this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN)
+            })
+        ),
+        asStream
+    );
+}
+
+async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn) {
+    if (!listQryFn) {
+        listQryFn = qry => qry;
+    }
+
+    return await _getCampaignStatistics(
+        campaign,
+        select,
+        unionQryFn,
+        qry => listQryFn(
+            qry.where(function() {
+                this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK)
+            })
+        ),
+        asStream
+    );
+}
+
+async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn) {
+    if (!listQryFn) {
+        listQryFn = qry => qry;
+    }
+
+    return await _getCampaignStatistics(
+        campaign,
+        select,
+        unionQryFn,
+        qry => listQryFn(
+            qry.where(function() {
+                this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK)
+            })
+        ),
+        asStream
+    );
+}
+
+async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) {
+    return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, false);
+}
+
+async function getCampaignOpenStatisticsStream(campaign, select, unionQryFn, listQryFn) {
+    return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, true);
+}
+
+async function getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn) {
+    return await _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn, false);
+}
+
+async function getCampaignClickStatisticsStream(campaign, select, unionQryFn, listQryFn) {
+    return await _getCampaignClickStatistics(campaign, select, unionQryFn, listQryFn, true);
+}
+
+async function getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn) {
+    return await _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn, false);
+}
+
+async function getCampaignLinkClickStatisticsStream(campaign, select, unionQryFn, listQryFn) {
+    return await _getCampaignLinkClickStatistics(campaign, select, unionQryFn, listQryFn, true);
+}
+
+
 
 
 module.exports.ReportState = ReportState;
@@ -205,4 +388,9 @@ module.exports.remove = remove;
 module.exports.updateFields = updateFields;
 module.exports.listByState = listByState;
 module.exports.bulkChangeState = bulkChangeState;
-module.exports.getCampaignResults = getCampaignResults;
+module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics;
+module.exports.getCampaignClickStatistics = getCampaignClickStatistics;
+module.exports.getCampaignLinkClickStatistics = getCampaignLinkClickStatistics;
+module.exports.getCampaignOpenStatisticsStream = getCampaignOpenStatisticsStream;
+module.exports.getCampaignClickStatisticsStream = getCampaignClickStatisticsStream;
+module.exports.getCampaignLinkClickStatisticsStream = getCampaignLinkClickStatisticsStream;

server/models/send-configurations.js

@@ -173,7 +173,6 @@ async function getSystemSendConfiguration() {
     return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId(), false);
 }
 
-module.exports.MailerType = MailerType;
 module.exports.hash = hash;
 module.exports.listDTAjax = listDTAjax;
 module.exports.listWithSendPermissionDTAjax = listWithSendPermissionDTAjax;

server/models/subscriptions.js

@@ -369,7 +369,7 @@ async function listTestUsersDTAjax(context, listCid, params) {
     });
 }
 
-async function list(context, listId, grouped = true, offset, limit) {
+async function list(context, listId, grouped, offset, limit) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
 

server/routes/reports.js

@@ -16,7 +16,7 @@ const fileSuffixes = {
 router.getAsync('/:id/download', passport.loggedIn, async (req, res) => {
     await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
 
-    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id, false);
 
     if (report.state == reports.ReportState.FINISHED) {
         const headers = {

server/services/executor.js

@@ -4,6 +4,7 @@
    that can chroot.
   */
 
+const config = require('config');
 const reportHelpers = require('../lib/report-helpers');
 const fork = require('child_process').fork;
 const path = require('path');
@@ -111,7 +112,7 @@ process.on('message', msg => {
         if (type === 'start-report-processor-worker') {
 
             const ids = privilegeHelpers.getConfigROUidGid();
-            spawnProcess(msg.tid, path.join(__dirname, '..', 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, '..', 'workers', 'reports'), ids.uid, ids.gid);
+            spawnProcess(msg.tid, path.join(__dirname, 'workers', 'reports', 'report-processor.js'), [msg.data.id], reportHelpers.getReportContentFile(msg.data), reportHelpers.getReportOutputFile(msg.data), path.join(__dirname, 'workers', 'reports'), ids.uid, ids.gid);
 
         } else if (type === 'stop-process') {
             const child = processes[msg.tid];
@@ -126,6 +127,10 @@ process.on('message', msg => {
     }
 });
 
+if (config.title) {
+    process.title = config.title + ': worker executor';
+}
+
 process.send({
     type: 'executor-started'
 });

server/services/feedcheck.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const config = require('config');
 const log = require('../lib/log');
 const knex = require('../lib/knex');
 const feedparser = require('feedparser-promised');
@@ -158,6 +159,10 @@ async function run() {
     setTimeout(run, dbCheckInterval);
 }
 
+if (config.title) {
+    process.title = config.title + ': feedcheck';
+}
+
 process.send({
     type: 'feedcheck-started'
 });

server/services/importer.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const config = require('config');
 const knex = require('../lib/knex');
 const path = require('path');
 const log = require('../lib/log');
@@ -402,6 +403,10 @@ process.on('message', msg => {
     }
 });
 
+if (config.title) {
+    process.title = config.title + ': importer';
+}
+
 process.send({
     type: 'importer-started'
 });

server/services/sender-master.js

@@ -333,6 +333,10 @@ async function init() {
         }
     });
 
+    if (config.title) {
+        process.title = config.title + ': sender/master';
+    }
+
     process.send({
         type: 'master-sender-started'
     });

server/services/sender-worker.js

@@ -65,6 +65,10 @@ process.on('message', msg => {
     }
 });
 
+if (config.title) {
+    process.title = config.title + ': sender/worker ' + workerId;
+}
+
 sendToMaster('worker-started');
 
 

server/services/workers/reports/config/default.yaml

@@ -1,13 +1,18 @@
 # Process title visible in monitoring logs and process listing
 title: mailtrain
 
-# Default language to use
-language: en
-
 log:
   # silly|verbose|info|http|warn|error|silent
   level: info
 
+# Default language to use
+defaultLanguage: en-US
+
+# Enabled languages
+enabledLanguages:
+- en-US
+- fk-FK
+
 mysql:
   host: localhost
   user: mailtrain
@@ -18,5 +23,5 @@ mysql:
   port: 3306
   charset: utf8mb4
   # The timezone configured on the MySQL server. This can be 'local', 'Z', or an offset in the form +HH:MM or -HH:MM
+  # If the MySQL server runs on the same server as Mailtrain, use 'local'
   timezone: local
-

server/services/workers/reports/report-processor.js

@@ -1,34 +1,33 @@
 'use strict';
 
-const reports = require('../../models/reports');
+const reports = require('../../../models/reports');
-const reportTemplates = require('../../models/report-templates');
+const reportTemplates = require('../../../models/report-templates');
-const lists = require('../../models/lists');
+const lists = require('../../../models/lists');
-const subscriptions = require('../../models/subscriptions');
+const subscriptions = require('../../../models/subscriptions');
-const campaigns = require('../../models/campaigns');
+const campaigns = require('../../../models/campaigns');
 const handlebars = require('handlebars');
-const handlebarsHelpers = require('../../lib/handlebars-helpers');
 const hbs = require('hbs');
 const vm = require('vm');
-const log = require('../../lib/log');
+const log = require('../../../lib/log');
 const fs = require('fs');
-const knex = require('../../lib/knex');
+const knex = require('../../../lib/knex');
-const contextHelpers = require('../../lib/context-helpers');
+const contextHelpers = require('../../../lib/context-helpers');
 
-
+const csvStringify = require('csv-stringify');
-handlebarsHelpers.registerHelpers(handlebars);
+const stream = require('stream');
 
 async function main() {
     try {
         const context = contextHelpers.getAdminContext();
 
         const userFieldGetters = {
-            'campaign': campaigns.getById,
+            'campaign': id => campaigns.getById(context, id, false, campaigns.Content.ALL),
             'list': id => lists.getById(context, id)
         };
 
         const reportId = Number(process.argv[2]);
 
-        const report = await reports.getByIdWithTemplate(context, reportId);
+        const report = await reports.getByIdWithTemplate(context, reportId, false);
 
         const inputs = {};
 
@@ -51,22 +50,42 @@ async function main() {
         }
 
         const campaignsProxy = {
-            getResults: (campaign, select, extra) => reports.getCampaignResults(context, campaign, select, extra),
+            getCampaignOpenStatistics: reports.getCampaignOpenStatistics,
-            getById: campaignId => campaigns.getById(context, campaignId)
+            getCampaignClickStatistics: reports.getCampaignClickStatistics,
+            getCampaignLinkClickStatistics: reports.getCampaignLinkClickStatistics,
+            getCampaignOpenStatisticsStream: reports.getCampaignOpenStatisticsStream,
+            getCampaignClickStatisticsStream: reports.getCampaignClickStatisticsStream,
+            getCampaignLinkClickStatisticsStream: reports.getCampaignLinkClickStatisticsStream,
+            getById: campaignId => campaigns.getById(context, campaignId, false, campaigns.Content.ALL)
         };
 
         const subscriptionsProxy = {
-            list: listId => subscriptions.list(context, listId)
+            list: (listId, grouped, offset, limit) => subscriptions.list(context, listId, grouped, offset, limit)
         };
 
         const sandbox = {
             console,
             campaigns: campaignsProxy,
             subscriptions: subscriptionsProxy,
+            stream,
             knex,
             process,
             inputs,
 
+            renderCsvFromStream: async (readable, opts) => {
+                const stringifier = csvStringify(opts);
+
+                const finished = new Promise((success, fail) => {
+                    stringifier.on('finish', () => success())
+                    stringifier.on('error', (err) => fail(err))
+                });
+
+                stringifier.pipe(process.stdout);
+                readable.pipe(stringifier);
+
+                await finished;
+            },
+
             render: data => {
                 const hbsTmpl = handlebars.compile(report.hbs);
                 const reportText = hbsTmpl(data);

shared/send-configurations.js

@@ -9,7 +9,8 @@ const MailerType = {
 const ZoneMTAType = {
     REGULAR: 0,
     WITH_HTTP_CONF: 1,
-    WITH_MAILTRAIN_HEADER_CONF: 2
+    WITH_MAILTRAIN_HEADER_CONF: 2,
+    BUILTIN: 3
 }
 
 function getSystemSendConfigurationId() {

zone-mta/.gitignore

@@ -0,0 +1 @@
+/config/builtin-zonemta.json