Fixes.

Added support for help text in custom fields.
Reimplemented the mechanism how campaign_messages are created.

client/src/campaigns/Statistics.js

@@ -73,7 +73,7 @@ export default class Statistics extends Component {
     render() {
         const t = this.props.t;
         const entity = this.state.entity;
-        const total = entity.subscriptionsToSend === undefined ? undefined : entity.subscriptionsToSend + entity.delivered;
+        const total = entity.total;
 
         const renderMetrics = (key, label, showZoomIn = true) => {
             const val = entity[key]

client/src/campaigns/Status.js

@@ -329,8 +329,6 @@ class SendControls extends Component {
 
         if (entity.status === CampaignStatus.IDLE || entity.status === CampaignStatus.PAUSED || (entity.status === CampaignStatus.SCHEDULED && entity.scheduled)) {
 
-            const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`;
-
             const timezoneColumns = [
                 { data: 0, title: t('Timezone') }
             ];
@@ -372,9 +370,9 @@ class SendControls extends Component {
                     </Form>
                     <ButtonRow className={campaignsStyles.sendButtonRow}>
                         {this.getFormValue('sendLater') ?
-                            <Button className="btn-primary" icon="play" label={(entity.status === CampaignStatus.SCHEDULED ? t('rescheduleSend') : t('scheduleSend')) + subscrInfo} onClickAsync={::this.confirmSchedule}/>
+                            <Button className="btn-primary" icon="play" label={entity.status === CampaignStatus.SCHEDULED ? t('rescheduleSend') : t('scheduleSend')} onClickAsync={::this.confirmSchedule}/>
                             :
-                            <Button className="btn-primary" icon="play" label={t('send') + subscrInfo} onClickAsync={::this.confirmStart}/>
+                            <Button className="btn-primary" icon="play" label={t('send')} onClickAsync={::this.confirmStart}/>
                         }
                         {entity.status === CampaignStatus.SCHEDULED && <Button className="btn-primary" icon="pause" label={t('Pause')} onClickAsync={::this.stopAsync}/>}
                         {entity.status === CampaignStatus.PAUSED && <Button className="btn-primary" icon="redo" label={t('reset')} onClickAsync={::this.resetAsync}/>}
@@ -413,15 +411,13 @@ class SendControls extends Component {
             );
 
         } else if (entity.status === CampaignStatus.FINISHED) {
-            const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`;
-
             return (
                 <div>{dialogs}
                     <AlignedRow label={t('sendStatus')}>
                         {t('allMessagesSent!HitContinueIfYouYouWant')}
                     </AlignedRow>
                     <ButtonRow>
-                        <Button className="btn-primary" icon="play" label={t('continue') + subscrInfo} onClickAsync={::this.confirmStart}/>
+                        <Button className="btn-primary" icon="play" label={t('continue')} onClickAsync={::this.confirmStart}/>
                         <Button className="btn-primary" icon="redo" label={t('reset')} onClickAsync={::this.resetAsync}/>
                         <LinkButton className="btn-secondary" icon="signal" label={t('viewStatistics')} to={`/campaigns/${entity.id}/statistics`}/>
                         {testButtons}

client/src/lists/fields/CUD.js

@@ -18,6 +18,7 @@ import {
     InputField,
     StaticField,
     TableSelect,
+    TextArea,
     withForm,
     withFormErrorHandlers
 } from '../../lib/form';
@@ -83,6 +84,10 @@ export default class CUD extends Component {
             data.default_value = '';
         }
 
+        if (data.help === null) {
+            data.help = '';
+        }
+
         data.isInGroup = data.group !== null;
 
         data.enumOptions = '';
@@ -124,6 +129,10 @@ export default class CUD extends Component {
             data.default_value = null;
         }
 
+        if (data.help.trim() === '') {
+            data.help = null;
+        }
+
         if (!data.isInGroup) {
             data.group = null;
         }
@@ -164,7 +173,7 @@ export default class CUD extends Component {
             data.orderManageBefore = Number.parseInt(data.orderManageBefore) || data.orderManageBefore;
         }
 
-        return filterData(data, ['name', 'key', 'default_value', 'type', 'group', 'settings',
+        return filterData(data, ['name', 'help', 'key', 'default_value', 'type', 'group', 'settings',
             'orderListBefore', 'orderSubscribeBefore', 'orderManageBefore']);
     }
 
@@ -178,6 +187,7 @@ export default class CUD extends Component {
                 type: 'text',
                 key: '',
                 default_value: '',
+                help: '',
                 group: null,
                 isInGroup: false,
                 renderTemplate: '',
@@ -501,6 +511,8 @@ export default class CUD extends Component {
 
                     <InputField id="key" label={t('mergeTag-1')}/>
 
+                    <TextArea id="help" label={t('Help text')}/>
+
                     {fieldSettings}
 
                     {type !== 'option' &&

client/src/lists/forms/CUD.js

@@ -494,7 +494,7 @@ export default class CUD extends Component {
                         { previewListId &&
                             <div>
                                 <AlignedRow>
-                                    <div className="help-block">
+                                    <div>
                                         <small>
                                             {t('noteTheseLinksAreSolelyForAQuickPreview')}
                                         </small>

client/static/subscription/form-input-style.css

@@ -199,11 +199,10 @@ textarea {
 
 /* --- Other ------------- */
 
-.help-block {
+.option-help-block {
     display: block;
-    font-size: .9em;
+    margin-left: 3px;
-    line-height: 1;
+    margin-bottom: 4px;
-    color: #999999;
 }
 
 form a {

server/lib/dbcheck.js

@@ -12,7 +12,7 @@ const pathlib = require('path');
 const Handlebars = require('handlebars');
 const bluebird = require('bluebird');
 
-const highestLegacySchemaVersion = 33;
+const highestLegacySchemaVersion = 34;
 
 const mysqlConfig = {
     multipleStatements: true

server/lib/helpers.js

@@ -1,10 +1,14 @@
 'use strict';
 
+const crypto = require('crypto');
+
 module.exports = {
     enforce,
     cleanupFromPost,
     filterObject,
-    castToInteger
+    castToInteger,
+    normalizeEmail,
+    hashEmail
 };
 
 function enforce(condition, message) {
@@ -36,4 +40,22 @@ function castToInteger(id, msg) {
     }
 
     return val;
-}
+}
+
+function normalizeEmail(email) {
+    const emailParts = email.split(/@/);
+
+    if (emailParts.length !== 2) {
+        return email;
+    }
+
+    const username = emailParts[0];
+    const domain = emailParts[1].toLowerCase();
+
+    return username + '@' + domain;
+}
+
+function hashEmail(email) {
+    return crypto.createHash('sha512').update(normalizeEmail(email)).digest("base64");
+}
+

server/lib/message-sender.js

@@ -22,7 +22,7 @@ const files = require('../models/files');
 const {getPublicUrl} = require('./urls');
 const blacklist = require('../models/blacklist');
 const libmime = require('libmime');
-const { enforce } = require('./helpers');
+const { enforce, hashEmail } = require('./helpers');
 const senders = require('./senders');
 
 const MessageType = {
@@ -187,8 +187,6 @@ class MessageSender {
             renderTags = true;
 
         } else if (campaign && campaign.source === CampaignSource.URL) {
-            const mergeTags = subData.mergeTags;
-
             const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
             for (const key in mergeTags) {
                 form[key] = mergeTags[key];
@@ -274,7 +272,7 @@ class MessageSender {
 
         Option #1
         - listId
-        - subscriptionId / email
+        - subscriptionId
         - mergeTags [optional, used only when campaign / html+text is provided]
 
         Option #2:
@@ -303,10 +301,6 @@ class MessageSender {
             if (subData.subscriptionId) {
                 listId = subData.listId;
                 subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, subData.subscriptionId);
-
-            } else if (subData.email) {
-                listId = subData.listId;
-                subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, subData.email);
             }
 
             list = this.listsById.get(listId);
@@ -474,7 +468,7 @@ class MessageSender {
 
         const result = {
             response,
-            response_id: responseId,
+            responseId: responseId,
             list,
             subscriptionGrouped,
             email
@@ -483,28 +477,28 @@ class MessageSender {
         return result;
     }
 
-    async sendRegularCampaignMessage(listId, email) {
+    async sendRegularCampaignMessage(campaignMessage) {
         enforce(this.type === MessageType.REGULAR);
 
-        // We insert into campaign_messages before the message is actually sent. This is to avoid multiple delivery
+        // We set the campaign_message to SENT before the message is actually sent. This is to avoid multiple delivery
-        // if by chance we run out of disk space and couldn't insert in the database after the message has been sent out
+        // if by chance we run out of disk space and couldn't change status in the database after the message has been sent out
-        const ids = await knex('campaign_messages').insert({
+        await knex('campaign_messages')
-            campaign: this.campaign.id,
+            .where({id: campaignMessage.id})
-            list: result.list.id,
+            .update({
-            subscription: result.subscriptionGrouped.id,
+                status: CampaignMessageStatus.SENT,
-            send_configuration: this.sendConfiguration.id,
+                updated: new Date()
-            status: CampaignMessageStatus.SENDING
+            });
-        });
-
-        const campaignMessageId = ids[0];
 
         let result;
         try {
-            result = await this._sendMessage({listId, email});
+            result = await this._sendMessage({listId: campaignMessage.list, subscriptionId: campaignMessage.subscription});
         } catch (err) {
             await knex('campaign_messages')
-                .where({id: campaignMessageId})
+                .where({id: campaignMessage.id})
-                .del();
+                .update({
+                    status: CampaignMessageStatus.SCHEDULED,
+                    updated: new Date()
+                });
 
             throw err;
         }
@@ -512,15 +506,12 @@ class MessageSender {
         enforce(result.list);
         enforce(result.subscriptionGrouped);
 
-        const now = new Date();
-
         await knex('campaign_messages')
-            .where({id: campaignMessageId})
+            .where({id: campaignMessage.id})
             .update({
-                status: CampaignMessageStatus.SENT,
                 response: result.response,
                 response_id: result.responseId,
-                updated: now
+                updated: new Date()
             });
 
         await knex('campaigns').where('id', this.campaign.id).increment('delivered');
@@ -600,16 +591,18 @@ async function sendQueuedMessage(queuedMessage) {
         }
     }
 
-    for (const attachment of msgData.attachments) {
+    if (msgData.attachments) {
-        if (attachment.id) { // This means that it is an attachment recorded in table files_campaign_attachment
+        for (const attachment of msgData.attachments) {
-            try {
+            if (attachment.id) { // This means that it is an attachment recorded in table files_campaign_attachment
-                // We ignore any errors here because we already sent the message. Thus we have to mark it as completed to avoid sending it again.
+                try {
-                await knex.transaction(async tx => {
+                    // We ignore any errors here because we already sent the message. Thus we have to mark it as completed to avoid sending it again.
-                    await files.unlockTx(tx, 'campaign', 'attachment', attachment.id);
+                    await knex.transaction(async tx => {
-                });
+                        await files.unlockTx(tx, 'campaign', 'attachment', attachment.id);
-            } catch (err) {
+                    });
-                log.error('MessageSender', `Error when unlocking attachment ${attachment.id} for ${result.email} (queuedId: ${queuedMessage.id})`);
+                } catch (err) {
-                log.verbose(err.stack);
+                    log.error('MessageSender', `Error when unlocking attachment ${attachment.id} for ${result.email} (queuedId: ${queuedMessage.id})`);
+                    log.verbose(err.stack);
+                }
             }
         }
     }

server/models/campaigns.js

@@ -390,11 +390,13 @@ async function getByIdTx(tx, context, id, withPermissions = true, content = Cont
 
         await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
 
-        const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id);
+        const totalRes = await tx('campaign_messages')
-        if (unsentQryGen) {
+            .where({campaign: id})
-            const res = await unsentQryGen(tx).count('* AS subscriptionsToSend').first();
+            .whereIn('status', [CampaignMessageStatus.SCHEDULED, CampaignMessageStatus.SENT,
-            entity.subscriptionsToSend = res.subscriptionsToSend;
+                CampaignMessageStatus.COMPLAINED, CampaignMessageStatus.UNSUBSCRIBED, CampaignMessageStatus.BOUNCED])
-        }
+            .count('* as count').first();
+
+        entity.total = totalRes.count;
 
     } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
         delete entity.data.sourceCustom;
@@ -699,7 +701,7 @@ async function getMessageByCid(messageCid, withVerpHostname = false) { // withVe
             .where(subscrTblName + '.cid', subscriptionCid)
             .where('campaigns.cid', campaignCid)
             .select([
-                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.status'
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.hash_email', 'campaign_messages.status'
             ])
             .first();
 
@@ -719,7 +721,7 @@ async function getMessageByResponseId(responseId) {
     return await knex('campaign_messages')
         .where('campaign_messages.response_id', responseId)
         .select([
-            'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.status'
+            'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.hash_email', 'campaign_messages.status'
         ])
         .first();
 }
@@ -754,7 +756,7 @@ async function changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaig
         .where('campaigns.cid', campaignCid)
         .where({subscription: subscriptionId, list: listId})
         .select([
-            'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.status'
+            'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.hash_email', 'campaign_messages.status'
         ])
         .first();
 
@@ -793,73 +795,34 @@ async function updateMessageResponse(context, message, response, responseId) {
     });
 }
 
-async function getSubscribersQueryGeneratorTx(tx, campaignId) {
+async function prepareCampaignMessages(campaignId) {
-    /*
+    const campaign = await getById(contextHelpers.getAdminContext(), campaignId, false);
-    This is supposed to produce queries like this:
 
-    select ... from `campaign_lists` inner join (
+    await knex('campaign_messages').where({campaign: campaignId, status: CampaignMessageStatus.SCHEDULED}).del();
-        select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (
-            (select `subscription__2`.`email`, 8 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__2` left join
-                (select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 2)
-                    as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__2`.`id` where `subscription__2`.`status` = 1)
-            UNION ALL
-            (select `subscription__1`.`email`, 9 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__1` left join
-                (select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 1)
-                    as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__1`.`id` where `subscription__1`.`status` = 1)
-        ) as `pending_subscriptions_all` where `sent` = false group by `email`)
-            as `pending_subscriptions` on `campaign_lists`.`id` = `pending_subscriptions`.`campaign_list_id` where `campaign_lists`.`campaign` = '1'
 
-     This was too much for Knex, so we partially construct these queries directly as strings;
+    for (const cpgList of campaign.lists) {
-     */
+        let addSegmentQuery;
-
+        await knex.transaction(async tx => {
-    const subsQrys = [];
+            addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
-    const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+        });
-
-    for (const cpgList of cpgLists) {
-        const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
         const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
 
-        const sqlQry = knex.from(subsTable)
+        const subsQry = knex.from(subsTable)
-            .leftJoin(
-                function () {
-                    return this.from('campaign_messages')
-                        .where('campaign_messages.campaign', campaignId)
-                        .where('campaign_messages.list', cpgList.list)
-                        .as('related_campaign_messages');
-                },
-                'related_campaign_messages.subscription', subsTable + '.id')
             .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
             .where(function() {
                 addSegmentQuery(this);
             })
-            .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('related_campaign_messages.id IS NOT NULL AS sent')])
+            .select([
+                'hash_email',
+                'id',
+                knex.raw('? AS campaign', [campaign.id]),
+                knex.raw('? AS list', [cpgList.list]),
+                knex.raw('? AS send_configuration', [campaign.send_configuration]),
+                knex.raw('? AS status', [CampaignMessageStatus.SCHEDULED])
+            ])
             .toSQL().toNative();
 
-        subsQrys.push(sqlQry);
+        await knex.raw('INSERT IGNORE INTO `campaign_messages` (`hash_email`, `subscription`, `campaign`, `list`, `send_configuration`, `status`) ' + subsQry.sql, subsQry.bindings);
-    }
-
-    if (subsQrys.length > 0) {
-        let subsQry;
-        const unsentWhere = ' where `sent` = false';
-
-        if (subsQrys.length === 1) {
-            const subsUnionSql = '(select `email`, `campaign_list_id`, `sent` from (' + subsQrys[0].sql + ') as `pending_subscriptions_all`' +  unsentWhere + ') as `pending_subscriptions`'
-            subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
-
-        } else {
-            const subsUnionSql = '(select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (' +
-                subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
-                ') as `pending_subscriptions_all`' +  unsentWhere + ' group by `email`) as `pending_subscriptions`';
-            const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
-            subsQry = knex.raw(subsUnionSql, subsUnionBindings);
-        }
-
-        return knx => knx.from('campaign_lists')
-            .where('campaign_lists.campaign', campaignId)
-            .innerJoin(subsQry, 'campaign_lists.id', 'pending_subscriptions.campaign_list_id');
-
-    } else {
-        return null;
     }
 }
 
@@ -1125,7 +1088,7 @@ module.exports.changeStatusByCampaignCidAndSubscriptionIdTx = changeStatusByCamp
 module.exports.changeStatusByMessage = changeStatusByMessage;
 module.exports.updateMessageResponse = updateMessageResponse;
 
-module.exports.getSubscribersQueryGeneratorTx = getSubscribersQueryGeneratorTx;
+module.exports.prepareCampaignMessages = prepareCampaignMessages;
 
 module.exports.start = start;
 module.exports.stop = stop;

server/models/fields.js

@@ -20,8 +20,8 @@ const {ListActivityType} = require('../../shared/activity-log');
 const activityLog = require('../lib/activity-log');
 
 
-const allowedKeysCreate = new Set(['name', 'key', 'default_value', 'type', 'group', 'settings']);
+const allowedKeysCreate = new Set(['name', 'help', 'key', 'default_value', 'type', 'group', 'settings']);
-const allowedKeysUpdate = new Set(['name', 'key', 'default_value', 'group', 'settings']);
+const allowedKeysUpdate = new Set(['name', 'help', 'key', 'default_value', 'group', 'settings']);
 const hashKeys = allowedKeysCreate;
 
 const fieldTypes = {};
@@ -304,7 +304,7 @@ async function getById(context, listId, id) {
 }
 
 async function listTx(tx, listId) {
-    return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'key', 'column', 'settings', 'group', 'default_value', 'order_list', 'order_subscribe', 'order_manage']).orderBy(knex.raw('-order_list'), 'desc').orderBy('id', 'asc');
+    return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'help', 'key', 'column', 'settings', 'group', 'default_value', 'order_list', 'order_subscribe', 'order_manage']).orderBy(knex.raw('-order_list'), 'desc').orderBy('id', 'asc');
 }
 
 async function list(context, listId) {
@@ -661,10 +661,11 @@ function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes group
         const entry = {
             name: fld.name,
             key: fld.key,
+            help: fld.help,
             field: fld,
             [type.getHbsType(fld)]: true,
             order_subscribe: fld.order_subscribe,
-            order_manage: fld.order_manage,
+            order_manage: fld.order_manage
         };
 
         if (!type.grouped && !type.enumerated) {
@@ -688,6 +689,7 @@ function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes group
                 options.push({
                     key: opt.key,
                     name: opt.name,
+                    help: opt.help,
                     value: isEnabled
                 });
             }
@@ -702,6 +704,7 @@ function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes group
                 options.push({
                     key: opt.key,
                     name: opt.label,
+                    help: opt.help,
                     value: value === opt.key
                 });
             }

server/models/subscriptions.js

@@ -11,10 +11,9 @@ const fields = require('./fields');
 const { SubscriptionSource, SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
 const { CampaignMessageStatus } = require('../../shared/campaigns');
 const segments = require('./segments');
-const { enforce, filterObject } = require('../lib/helpers');
+const { enforce, filterObject, hashEmail, normalizeEmail } = require('../lib/helpers');
 const moment = require('moment');
 const { formatDate, formatBirthday } = require('../../shared/date');
-const crypto = require('crypto');
 const campaigns = require('./campaigns');
 const lists = require('./lists');
 
@@ -85,7 +84,6 @@ fieldTypes.option = {
 };
 
 
-
 function getSubscriptionTableName(listId) {
     return `subscription__${listId}`;
 }
@@ -232,7 +230,12 @@ async function getById(context, listId, id, grouped = true) {
 }
 
 async function getByEmail(context, listId, email, grouped = true) {
-    return await _getBy(context, listId, 'email', email, grouped);
+    const result = await _getBy(context, listId, 'hash_email', hashEmail(email), grouped);
+    if (result.email === null) {
+        throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+    }
+    enforce(normalizeEmail(email) === normalizeEmail(result.email));
+    return result;
 }
 
 async function getByCid(context, listId, cid, grouped = true) {
@@ -486,7 +489,7 @@ async function serverValidate(context, listId, data) {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
 
         if (data.email) {
-            const existingKeyQuery = tx(getSubscriptionTableName(listId)).where('email', data.email);
+            const existingKeyQuery = tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(data.email)).whereNotNull('email');
 
             if (data.id) {
                 existingKeyQuery.whereNot('id', data.id);
@@ -539,10 +542,6 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
     }
 }
 
-function hashEmail(email) {
-    return crypto.createHash('sha512').update(email).digest("base64");
-}
-
 function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
     if ('email' in subscription) {
         subscription.hash_email = hashEmail(subscription.email);
@@ -865,7 +864,7 @@ async function getListsWithEmail(context, email) {
 
         for (const list of lsts) {
             await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
-            const entity = await tx(getSubscriptionTableName(list.id)).where('email', email).first();
+            const entity = await tx(getSubscriptionTableName(list.id)).where('hash_email', hashEmail(email)).whereNotNull('email').first();
             if (entity) {
                 result.push(list);
             }
@@ -901,4 +900,4 @@ module.exports.updateAddressAndGet = updateAddressAndGet;
 module.exports.updateManaged = updateManaged;
 module.exports.getListsWithEmail = getListsWithEmail;
 module.exports.changeStatusTx = changeStatusTx;
-module.exports.purgeSensitiveData = purgeSensitiveData;
+module.exports.purgeSensitiveData = purgeSensitiveData;

server/services/sender-master.js

@@ -5,7 +5,7 @@ const fork = require('../lib/fork').fork;
 const log = require('../lib/log');
 const path = require('path');
 const knex = require('../lib/knex');
-const {CampaignStatus, CampaignType} = require('../../shared/campaigns');
+const {CampaignStatus, CampaignType, CampaignMessageStatus} = require('../../shared/campaigns');
 const campaigns = require('../models/campaigns');
 const builtinZoneMta = require('../lib/builtin-zone-mta');
 const {CampaignActivityType} = require('../../shared/activity-log');
@@ -62,10 +62,10 @@ const workerBatchSize = 10;
 const sendConfigurationIdByCampaignId = new Map(); // campaignId -> sendConfigurationId
 const sendConfigurationStatuses = new Map(); // sendConfigurationId -> {retryCount, postponeTill}
 
-const sendConfigurationMessageQueue = new Map(); // sendConfigurationId -> [{queuedMessage}]
+const sendConfigurationMessageQueue = new Map(); // sendConfigurationId -> [queuedMessage]
-const campaignMessageQueue = new Map(); // campaignId -> [{listId, email}]
+const campaignMessageQueue = new Map(); // campaignId -> [campaignMessage]
 
-const workAssignment = new Map(); // workerId -> { type: WorkAssignmentType.CAMPAIGN, campaignId, messages: [{listId, email} } / { type: WorkAssignmentType.QUEUED, sendConfigurationId, messages: [{queuedMessage}] }
+const workAssignment = new Map(); // workerId -> { type: WorkAssignmentType.CAMPAIGN, campaignId, messages: [campaignMessage] / { type: WorkAssignmentType.QUEUED, sendConfigurationId, messages: [queuedMessage] }
 
 const WorkAssignmentType = {
     CAMPAIGN: 0,
@@ -330,7 +330,10 @@ async function processCampaign(campaignId) {
         }
     }
 
+
     try {
+        await campaigns.prepareCampaignMessages(campaignId);
+
         while (true) {
             const cpg = await knex('campaigns').where('id', campaignId).first();
 
@@ -350,53 +353,39 @@ async function processCampaign(campaignId) {
                 return await finish(true, CampaignStatus.SCHEDULED);
             }
 
-            let qryGen;
+            let messagesInProcessing = [...msgQueue];
-            await knex.transaction(async tx => {
+            for (const wa of workAssignment.values()) {
-                qryGen = await campaigns.getSubscribersQueryGeneratorTx(tx, campaignId);
+                if (wa.type === WorkAssignmentType.CAMPAIGN && wa.campaignId === campaignId) {
-            });
+                    messagesInProcessing = messagesInProcessing.concat(wa.messages);
+                }
+            }
 
-            if (qryGen) {
+            const subs = await knex('campaign_messages')
-                let messagesInProcessing = [...msgQueue];
+                .where({status: CampaignMessageStatus.SCHEDULED})
-                for (const wa of workAssignment.values()) {
+                .whereNotIn('hash_email', messagesInProcessing.map(x => x.hash_email))
-                    if (wa.type === WorkAssignmentType.CAMPAIGN && wa.campaignId === campaignId) {
+                .limit(retrieveBatchSize);
-                        messagesInProcessing = messagesInProcessing.concat(wa.messages);
+
-                    }
+            if (subs.length === 0) {
+                if (isCompleted()) {
+                    return await finish(false, CampaignStatus.FINISHED);
+
+                } else {
+                    await finish(false);
+
+                    // At this point, there might be messages that re-appeared because sending failed.
+                    continue;
                 }
 
-                const qry = qryGen(knex)
+            }
-                    .whereNotIn('pending_subscriptions.email', messagesInProcessing.map(x => x.email))
-                    .select(['pending_subscriptions.email', 'campaign_lists.list'])
-                    .limit(retrieveBatchSize);
-                const subs = await qry;
 
-                if (subs.length === 0) {
+            for (const sub of subs) {
-                    if (isCompleted()) {
+                msgQueue.push(sub);
-                        return await finish(false, CampaignStatus.FINISHED);
+            }
 
-                    } else {
+            notifier.notify('workAvailable');
-                        await finish(false);
 
-                        // At this point, there might be messages that re-appeared because sending failed.
+            while (msgQueue.length > 0) {
-                        continue;
+                await notifier.waitFor(`campaignMessageQueueEmpty:${campaignId}`);
-                    }
-
-                }
-
-                for (const sub of subs) {
-                    msgQueue.push({
-                        listId: sub.list,
-                        email: sub.email
-                    });
-                }
-
-                notifier.notify('workAvailable');
-
-                while (msgQueue.length > 0) {
-                    await notifier.waitFor(`campaignMessageQueueEmpty:${campaignId}`);
-                }
-
-            } else {
-                return await finish(false, CampaignStatus.FINISHED);
             }
         }
     } catch (err) {
@@ -526,7 +515,7 @@ async function processQueuedBySendConfiguration(sendConfigurationId) {
                 }
             }
 
-            const messageIdsInProcessing = messagesInProcessing.map(x => x.queuedMessage.id);
+            const messageIdsInProcessing = messagesInProcessing.map(x => x.id);
 
             const rows = await knex('queued')
                 .orderByRaw(`FIELD(type, ${MessageType.TRIGGERED}, ${MessageType.API_TRANSACTIONAL}, ${MessageType.TEST}, ${MessageType.SUBSCRIPTION}) DESC, id ASC`) // This orders messages in the following order MessageType.SUBSCRIPTION, MessageType.TEST, MessageType.API_TRANSACTIONAL and MessageType.TRIGGERED
@@ -561,9 +550,7 @@ async function processQueuedBySendConfiguration(sendConfigurationId) {
 
                 } else {
                     row.data = JSON.parse(row.data);
-                    msgQueue.push({
+                    msgQueue.push(row);
-                        queuedMessage: row
-                    });
                 }
             }
 

server/services/sender-worker.js

@@ -22,20 +22,20 @@ async function processCampaignMessages(campaignId, messages) {
 
     let withErrors = false;
 
-    for (const msg of messages) {
+    for (const campaignMessage of messages) {
         try {
-            await cs.sendRegularCampaignMessage(msg.listId, msg.email);
+            await cs.sendRegularCampaignMessage(campaignMessage);
 
-            log.verbose('Senders', 'Message sent and status updated for %s:%s', msg.listId, msg.email);
+            log.verbose('Senders', 'Message sent and status updated for %s:%s', campaignMessage.list, campaignMessage.subscription);
         } catch (err) {
 
             if (err instanceof mailers.SendConfigurationError) {
-                log.error('Senders', `Sending message to ${msg.listId}:${msg.email} failed with error: ${err.message}. Will retry the message if within retention interval.`);
+                log.error('Senders', `Sending message to ${campaignMessage.list}:${campaignMessage.subscription} failed with error: ${err.message}. Will retry the message if within retention interval.`);
                 withErrors = true;
                 break;
 
             } else {
-                log.error('Senders', `Sending message to ${msg.listId}:${msg.email} failed with error: ${err.message}.`);
+                log.error('Senders', `Sending message to ${campaignMessage.list}:${campaignMessage.subscription} failed with error: ${err.message}.`);
                 log.verbose(err.stack);
             }
         }
@@ -56,8 +56,7 @@ async function processQueuedMessages(sendConfigurationId, messages) {
 
     let withErrors = false;
 
-    for (const msg of messages) {
+    for (const queuedMessage of messages) {
-        const queuedMessage = msg.queuedMessage;
 
         const msgData = queuedMessage.data;
         let target = '';

server/setup/knex/migrations/20170506102634_v1_to_v2.js

@@ -6,10 +6,9 @@ const {TagLanguages} = require('../../../../shared/templates');
 const {getGlobalNamespaceId} = require('../../../../shared/namespaces');
 const {getAdminId} = require('../../../../shared/users');
 const { MailerType, ZoneMTAType, getSystemSendConfigurationId, getSystemSendConfigurationCid } = require('../../../../shared/send-configurations');
-const { enforce } = require('../../../lib/helpers');
+const { enforce, hashEmail} = require('../../../lib/helpers');
 const { EntityVals: TriggerEntityVals, EventVals: TriggerEventVals } = require('../../../../shared/triggers');
 const { SubscriptionSource } = require('../../../../shared/lists');
-const crypto = require('crypto');
 const {DOMParser, XMLSerializer} = require('xmldom');
 const log = require('../../../lib/log');
 
@@ -271,7 +270,7 @@ async function migrateSubscriptions(knex) {
 
             if (rows.length > 0) {
                 for await (const subscription of rows) {
-                    subscription.hash_email = crypto.createHash('sha512').update(subscription.email).digest("base64");
+                    subscription.hash_email = hashEmail(subscription.email);
                     subscription.source_email = subscription.imported ? SubscriptionSource.IMPORTED_V1 : SubscriptionSource.NOT_IMPORTED_V1;
                     for (const field of fields) {
                         if (field.column != null) {
@@ -417,6 +416,7 @@ async function migrateCustomFields(knex) {
     });
 
     await knex.schema.table('custom_fields', table => {
+        table.renameColumn('description', 'help');
         table.dropColumn('visible');
     });
 

server/setup/knex/migrations/20190722110000_hash_email.js

@@ -0,0 +1,17 @@
+exports.up = (knex, Promise) => (async() => {
+    await knex.schema.raw('ALTER TABLE `campaign_messages` ADD `hash_email` char(88) CHARACTER SET ascii');
+    await knex.schema.raw('ALTER TABLE `campaign_messages` ADD UNIQUE KEY `campaign_hash_email` (`campaign`, `hash_email`)');
+    await knex.schema.raw('ALTER TABLE `campaign_messages` DROP KEY `created`');
+    await knex.schema.raw('ALTER TABLE `campaign_links` DROP KEY `created_index`');
+
+    const lists = await knex('lists');
+    for (const list of lists) {
+        await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` MODIFY `hash_email` char(88) CHARACTER SET ascii');
+        await knex.raw('update `campaign_messages` inner join `subscription__' + list.id + '` on `campaign_messages`.`list`=' + list.id + ' and `campaign_messages`.`subscription`=`subscription__' + list.id + '`.`id` set `campaign_messages`.`hash_email`=`subscription__' + list.id + '`.`hash_email`');
+    }
+
+    await knex('campaign_messages').whereNull('hash_email').del();
+})();
+
+exports.down = (knex, Promise) => (async() => {
+})();

server/setup/knex/migrations/20190722150000_ensure_help_column_in_custom_fields.js

@@ -0,0 +1,7 @@
+exports.up = (knex, Promise) => (async() => {
+    // This is to provide upgrade path to stable to those that already have beta installed.
+    await knex.schema.raw('ALTER TABLE `custom_fields` ADD COLUMN `help` text AFTER `name`');
+})();
+
+exports.down = (knex, Promise) => (async() => {
+})();

server/setup/sql/upgrade-00034.sql

@@ -0,0 +1,11 @@
+# Header section
+# Define incrementing schema version number
+SET @schema_version = '34';
+
+# Add template field for group elements
+ALTER TABLE `custom_fields` ADD COLUMN `description` text AFTER `name`;
+
+# Footer section
+LOCK TABLES `settings` WRITE;
+INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
+UNLOCK TABLES;

server/views/subscription/partials/subscription-custom-fields.hbs

@@ -11,6 +11,7 @@
             {{else}}
                 <input type="email" name="EMAIL" id="email" placeholder="" value="{{../email}}" required>
             {{/if}}
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -18,6 +19,7 @@
         <div class="form-group text {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="text" name="{{key}}" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -25,6 +27,7 @@
         <div class="form-group number {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="number" name="{{key}}" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -32,6 +35,7 @@
         <div class="form-group url {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="url" name="{{key}}" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -39,6 +43,7 @@
         <div class="form-group longtext {{key}}">
             <label for="{{key}}">{{name}}</label>
             <textarea rows="3" name="{{key}}">{{value}}</textarea>
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -46,6 +51,7 @@
         <div class="form-group json {{key}}">
             <label for="{{key}}">{{name}}</label>
             <textarea class="gpg-text" rows="3" name="{{key}}" placeholder="{&quot;data&quot;:&quot;value&quot;}">{{value}}</textarea>
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -56,6 +62,7 @@
                 <input type="checkbox" name="{{key}}" value="1" {{#if value}} checked {{/if}}> {{field.settings.checkedLabel}}
             </label>
             <input type="hidden" value="0" name="{{key}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -66,9 +73,8 @@
                 <button class="btn-download-pubkey" type="submit" form="download-pubkey">{{#translate}}downloadSignatureVerificationKey{{/translate}}</button>
             {{/if}}
             <textarea class="form-control gpg-text" rows="4" name="{{key}}" placeholder="{{#translate}}beginsWithAnd#39BeginPgpPublicKeyBloc{{/translate}}">{{value}}</textarea>
-            <span class="help-block">
+            <small class="form-text text-muted">{{#translate}}insertYourGpgPublicKeyHereToEncrypt{{/translate}}</small>
-                {{#translate}}insertYourGpgPublicKeyHereToEncrypt{{/translate}}
+            <small class="form-text text-muted">{{help}}</small>
-            </span>
         </div>
     {{/if}}
 
@@ -76,6 +82,7 @@
         <div class="form-group date fm-date-us {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="text" name="{{key}}" placeholder="MM/DD/YYYY" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -83,6 +90,7 @@
         <div class="form-group date fm-date-eur {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="text" name="{{key}}" placeholder="DD/MM/YYYY" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -90,6 +98,7 @@
         <div class="form-group date fm-birthday-us {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="text" name="{{key}}" placeholder="MM/DD" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -97,6 +106,7 @@
         <div class="form-group date fm-birthday-eur {{key}}">
             <label for="{{key}}">{{name}}</label>
             <input type="text" name="{{key}}" placeholder="DD/MM" value="{{value}}">
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -111,6 +121,7 @@
                     <option value="{{key}}" {{#if value}} selected {{/if}}>{{name}}</option>
                 {{/each}}
             </select>
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -122,6 +133,7 @@
                     <input type="radio" name="{{../key}}" value="{{key}}" {{#if value}} checked {{/if}}> {{name}}
                 </label>
             {{/each}}
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -133,6 +145,7 @@
                 <label class="label-checkbox">
                     <input type="checkbox" name="{{key}}" value="1" {{#if value}} checked {{/if}}> {{name}}
                 </label>
+                <small class="option-help-block form-text text-muted">{{help}}</small>
             {{/each}}
         </div>
     {{/if}}
@@ -148,6 +161,7 @@
                     <option value="{{key}}" {{#if value}} selected {{/if}}>{{name}}</option>
                 {{/each}}
             </select>
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 
@@ -159,6 +173,7 @@
                     <input type="radio" name="{{../key}}" value="{{key}}" {{#if value}} checked {{/if}}> {{name}}
                 </label>
             {{/each}}
+            <small class="form-text text-muted">{{help}}</small>
         </div>
     {{/if}}
 

shared/campaigns.js

@@ -51,7 +51,7 @@ const CampaignMessageStatus = {
     UNSUBSCRIBED: 2,
     BOUNCED: 3,
     COMPLAINED: 4,
-    SENDING: 5,
+    SCHEDULED: 5,
 
     MAX: 5
 };