public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fixes.

Browse source 
Added support for help text in custom fields.
Reimplemented the mechanism how campaign_messages are created.
This commit is contained in:
Tomas Bures 2019-07-22 23:54:24 +05:30
parent 025600e818
commit 4e4b77ca84
19 changed files with 223 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
client/src/campaigns/Statistics.js
View file

@ -73,7 +73,7 @@ export default class Statistics extends Component {
render() {
const t = this.props.t;
const entity = this.state.entity;
const total = entity.subscriptionsToSend === undefined ? undefined : entity.subscriptionsToSend + entity.delivered;
const total = entity.total;
const renderMetrics = (key, label, showZoomIn = true) => {
const val = entity[key]

10
client/src/campaigns/Status.js
View file

@ -329,8 +329,6 @@ class SendControls extends Component {
if (entity.status === CampaignStatus.IDLE || entity.status === CampaignStatus.PAUSED || (entity.status === CampaignStatus.SCHEDULED && entity.scheduled)) {
const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`;
const timezoneColumns = [
{ data: 0, title: t('Timezone') }
];
@ -372,9 +370,9 @@ class SendControls extends Component {
</Form>
<ButtonRow className={campaignsStyles.sendButtonRow}>
{this.getFormValue('sendLater') ?
<Button className="btn-primary" icon="play" label={(entity.status === CampaignStatus.SCHEDULED ? t('rescheduleSend') : t('scheduleSend')) + subscrInfo} onClickAsync={::this.confirmSchedule}/>
<Button className="btn-primary" icon="play" label={entity.status === CampaignStatus.SCHEDULED ? t('rescheduleSend') : t('scheduleSend')} onClickAsync={::this.confirmSchedule}/>
:
<Button className="btn-primary" icon="play" label={t('send') + subscrInfo} onClickAsync={::this.confirmStart}/>
<Button className="btn-primary" icon="play" label={t('send')} onClickAsync={::this.confirmStart}/>
}
{entity.status === CampaignStatus.SCHEDULED && <Button className="btn-primary" icon="pause" label={t('Pause')} onClickAsync={::this.stopAsync}/>}
{entity.status === CampaignStatus.PAUSED && <Button className="btn-primary" icon="redo" label={t('reset')} onClickAsync={::this.resetAsync}/>}
@ -413,15 +411,13 @@ class SendControls extends Component {
);
} else if (entity.status === CampaignStatus.FINISHED) {
const subscrInfo = entity.subscriptionsToSend === undefined ? '' : ` (${entity.subscriptionsToSend} ${t('subscribers-1')})`;
return (
<div>{dialogs}
<AlignedRow label={t('sendStatus')}>
{t('allMessagesSent!HitContinueIfYouYouWant')}
</AlignedRow>
<ButtonRow>
<Button className="btn-primary" icon="play" label={t('continue') + subscrInfo} onClickAsync={::this.confirmStart}/>
<Button className="btn-primary" icon="play" label={t('continue')} onClickAsync={::this.confirmStart}/>
<Button className="btn-primary" icon="redo" label={t('reset')} onClickAsync={::this.resetAsync}/>
<LinkButton className="btn-secondary" icon="signal" label={t('viewStatistics')} to={`/campaigns/${entity.id}/statistics`}/>
{testButtons}

14
client/src/lists/fields/CUD.js
View file

@ -18,6 +18,7 @@ import {
InputField,
StaticField,
TableSelect,
TextArea,
withForm,
withFormErrorHandlers
} from '../../lib/form';
@ -83,6 +84,10 @@ export default class CUD extends Component {
data.default_value = '';
}
if (data.help === null) {
data.help = '';
}
data.isInGroup = data.group !== null;
data.enumOptions = '';
@ -124,6 +129,10 @@ export default class CUD extends Component {
data.default_value = null;
}
if (data.help.trim() === '') {
data.help = null;
}
if (!data.isInGroup) {
data.group = null;
}
@ -164,7 +173,7 @@ export default class CUD extends Component {
data.orderManageBefore = Number.parseInt(data.orderManageBefore) || data.orderManageBefore;
}
return filterData(data, ['name', 'key', 'default_value', 'type', 'group', 'settings',
return filterData(data, ['name', 'help', 'key', 'default_value', 'type', 'group', 'settings',
'orderListBefore', 'orderSubscribeBefore', 'orderManageBefore']);
}
@ -178,6 +187,7 @@ export default class CUD extends Component {
type: 'text',
key: '',
default_value: '',
help: '',
group: null,
isInGroup: false,
renderTemplate: '',
@ -501,6 +511,8 @@ export default class CUD extends Component {
<InputField id="key" label={t('mergeTag-1')}/>
<TextArea id="help" label={t('Help text')}/>
{fieldSettings}
{type !== 'option' &&

2
client/src/lists/forms/CUD.js
View file

@ -494,7 +494,7 @@ export default class CUD extends Component {
{ previewListId &&
<div>
<AlignedRow>
<div className="help-block">
<div>
<small>
{t('noteTheseLinksAreSolelyForAQuickPreview')}
</small>

7
client/static/subscription/form-input-style.css
View file

@ -199,11 +199,10 @@ textarea {
/* --- Other ------------- */
.help-block {
.option-help-block {
display: block;
font-size: .9em;
line-height: 1;
color: #999999;
margin-left: 3px;
margin-bottom: 4px;
}
form a {

2
server/lib/dbcheck.js
View file

@ -12,7 +12,7 @@ const pathlib = require('path');
const Handlebars = require('handlebars');
const bluebird = require('bluebird');
const highestLegacySchemaVersion = 33;
const highestLegacySchemaVersion = 34;
const mysqlConfig = {
multipleStatements: true

24
server/lib/helpers.js
View file

@ -1,10 +1,14 @@
'use strict';
const crypto = require('crypto');
module.exports = {
enforce,
cleanupFromPost,
filterObject,
castToInteger
castToInteger,
normalizeEmail,
hashEmail
};
function enforce(condition, message) {
@ -37,3 +41,21 @@ function castToInteger(id, msg) {
return val;
}
function normalizeEmail(email) {
const emailParts = email.split(/@/);
if (emailParts.length !== 2) {
return email;
}
const username = emailParts[0];
const domain = emailParts[1].toLowerCase();
return username + '@' + domain;
}
function hashEmail(email) {
return crypto.createHash('sha512').update(normalizeEmail(email)).digest("base64");
}

49
server/lib/message-sender.js
View file

@ -22,7 +22,7 @@ const files = require('../models/files');
const {getPublicUrl} = require('./urls');
const blacklist = require('../models/blacklist');
const libmime = require('libmime');
const { enforce } = require('./helpers');
const { enforce, hashEmail } = require('./helpers');
const senders = require('./senders');
const MessageType = {
@ -187,8 +187,6 @@ class MessageSender {
renderTags = true;
} else if (campaign && campaign.source === CampaignSource.URL) {
const mergeTags = subData.mergeTags;
const form = tools.getMessageLinks(campaign, list, subscriptionGrouped);
for (const key in mergeTags) {
form[key] = mergeTags[key];
@ -274,7 +272,7 @@ class MessageSender {
Option #1
- listId
- subscriptionId / email
- subscriptionId
- mergeTags [optional, used only when campaign / html+text is provided]
Option #2:
@ -303,10 +301,6 @@ class MessageSender {
if (subData.subscriptionId) {
listId = subData.listId;
subscriptionGrouped = await subscriptions.getById(contextHelpers.getAdminContext(), listId, subData.subscriptionId);
} else if (subData.email) {
listId = subData.listId;
subscriptionGrouped = await subscriptions.getByEmail(contextHelpers.getAdminContext(), listId, subData.email);
}
list = this.listsById.get(listId);
@ -474,7 +468,7 @@ class MessageSender {
const result = {
response,
response_id: responseId,
responseId: responseId,
list,
subscriptionGrouped,
email
@ -483,28 +477,28 @@ class MessageSender {
return result;
}
async sendRegularCampaignMessage(listId, email) {
async sendRegularCampaignMessage(campaignMessage) {
enforce(this.type === MessageType.REGULAR);
// We insert into campaign_messages before the message is actually sent. This is to avoid multiple delivery
// if by chance we run out of disk space and couldn't insert in the database after the message has been sent out
const ids = await knex('campaign_messages').insert({
campaign: this.campaign.id,
list: result.list.id,
subscription: result.subscriptionGrouped.id,
send_configuration: this.sendConfiguration.id,
status: CampaignMessageStatus.SENDING
// We set the campaign_message to SENT before the message is actually sent. This is to avoid multiple delivery
// if by chance we run out of disk space and couldn't change status in the database after the message has been sent out
await knex('campaign_messages')
.where({id: campaignMessage.id})
.update({
status: CampaignMessageStatus.SENT,
updated: new Date()
});
const campaignMessageId = ids[0];
let result;
try {
result = await this._sendMessage({listId, email});
result = await this._sendMessage({listId: campaignMessage.list, subscriptionId: campaignMessage.subscription});
} catch (err) {
await knex('campaign_messages')
.where({id: campaignMessageId})
.del();
.where({id: campaignMessage.id})
.update({
status: CampaignMessageStatus.SCHEDULED,
updated: new Date()
});
throw err;
}
@ -512,15 +506,12 @@ class MessageSender {
enforce(result.list);
enforce(result.subscriptionGrouped);
const now = new Date();
await knex('campaign_messages')
.where({id: campaignMessageId})
.where({id: campaignMessage.id})
.update({
status: CampaignMessageStatus.SENT,
response: result.response,
response_id: result.responseId,
updated: now
updated: new Date()
});
await knex('campaigns').where('id', this.campaign.id).increment('delivered');
@ -600,6 +591,7 @@ async function sendQueuedMessage(queuedMessage) {
}
}
if (msgData.attachments) {
for (const attachment of msgData.attachments) {
if (attachment.id) { // This means that it is an attachment recorded in table files_campaign_attachment
try {
@ -613,6 +605,7 @@ async function sendQueuedMessage(queuedMessage) {
}
}
}
}
}
async function dropQueuedMessage(queuedMessage) {

95
server/models/campaigns.js
View file

@ -390,11 +390,13 @@ async function getByIdTx(tx, context, id, withPermissions = true, content = Cont
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id);
if (unsentQryGen) {
const res = await unsentQryGen(tx).count('* AS subscriptionsToSend').first();
entity.subscriptionsToSend = res.subscriptionsToSend;
}
const totalRes = await tx('campaign_messages')
.where({campaign: id})
.whereIn('status', [CampaignMessageStatus.SCHEDULED, CampaignMessageStatus.SENT,
CampaignMessageStatus.COMPLAINED, CampaignMessageStatus.UNSUBSCRIBED, CampaignMessageStatus.BOUNCED])
.count('* as count').first();
entity.total = totalRes.count;
} else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
delete entity.data.sourceCustom;
@ -699,7 +701,7 @@ async function getMessageByCid(messageCid, withVerpHostname = false) { // withVe
.where(subscrTblName + '.cid', subscriptionCid)
.where('campaigns.cid', campaignCid)
.select([
'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.status'
'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.hash_email', 'campaign_messages.status'
])
.first();
@ -719,7 +721,7 @@ async function getMessageByResponseId(responseId) {
return await knex('campaign_messages')
.where('campaign_messages.response_id', responseId)
.select([
'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.status'
'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.hash_email', 'campaign_messages.status'
])
.first();
}
@ -754,7 +756,7 @@ async function changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaig
.where('campaigns.cid', campaignCid)
.where({subscription: subscriptionId, list: listId})
.select([
'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.status'
'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'campaign_messages.subscription', 'campaign_messages.hash_email', 'campaign_messages.status'
])
.first();
@ -793,73 +795,34 @@ async function updateMessageResponse(context, message, response, responseId) {
});
}
async function getSubscribersQueryGeneratorTx(tx, campaignId) {
/*
This is supposed to produce queries like this:
async function prepareCampaignMessages(campaignId) {
const campaign = await getById(contextHelpers.getAdminContext(), campaignId, false);
select ... from `campaign_lists` inner join (
select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (
(select `subscription__2`.`email`, 8 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__2` left join
(select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 2)
as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__2`.`id` where `subscription__2`.`status` = 1)
UNION ALL
(select `subscription__1`.`email`, 9 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__1` left join
(select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 1)
as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__1`.`id` where `subscription__1`.`status` = 1)
) as `pending_subscriptions_all` where `sent` = false group by `email`)
as `pending_subscriptions` on `campaign_lists`.`id` = `pending_subscriptions`.`campaign_list_id` where `campaign_lists`.`campaign` = '1'
await knex('campaign_messages').where({campaign: campaignId, status: CampaignMessageStatus.SCHEDULED}).del();
This was too much for Knex, so we partially construct these queries directly as strings;
*/
const subsQrys = [];
const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
for (const cpgList of cpgLists) {
const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
for (const cpgList of campaign.lists) {
let addSegmentQuery;
await knex.transaction(async tx => {
addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
});
const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
const sqlQry = knex.from(subsTable)
.leftJoin(
function () {
return this.from('campaign_messages')
.where('campaign_messages.campaign', campaignId)
.where('campaign_messages.list', cpgList.list)
.as('related_campaign_messages');
},
'related_campaign_messages.subscription', subsTable + '.id')
const subsQry = knex.from(subsTable)
.where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
.where(function() {
addSegmentQuery(this);
})
.select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('related_campaign_messages.id IS NOT NULL AS sent')])
.select([
'hash_email',
'id',
knex.raw('? AS campaign', [campaign.id]),
knex.raw('? AS list', [cpgList.list]),
knex.raw('? AS send_configuration', [campaign.send_configuration]),
knex.raw('? AS status', [CampaignMessageStatus.SCHEDULED])
])
.toSQL().toNative();
subsQrys.push(sqlQry);
}
if (subsQrys.length > 0) {
let subsQry;
const unsentWhere = ' where `sent` = false';
if (subsQrys.length === 1) {
const subsUnionSql = '(select `email`, `campaign_list_id`, `sent` from (' + subsQrys[0].sql + ') as `pending_subscriptions_all`' + unsentWhere + ') as `pending_subscriptions`'
subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
} else {
const subsUnionSql = '(select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (' +
subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
') as `pending_subscriptions_all`' + unsentWhere + ' group by `email`) as `pending_subscriptions`';
const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
subsQry = knex.raw(subsUnionSql, subsUnionBindings);
}
return knx => knx.from('campaign_lists')
.where('campaign_lists.campaign', campaignId)
.innerJoin(subsQry, 'campaign_lists.id', 'pending_subscriptions.campaign_list_id');
} else {
return null;
await knex.raw('INSERT IGNORE INTO `campaign_messages` (`hash_email`, `subscription`, `campaign`, `list`, `send_configuration`, `status`) ' + subsQry.sql, subsQry.bindings);
}
}
@ -1125,7 +1088,7 @@ module.exports.changeStatusByCampaignCidAndSubscriptionIdTx = changeStatusByCamp
module.exports.changeStatusByMessage = changeStatusByMessage;
module.exports.updateMessageResponse = updateMessageResponse;
module.exports.getSubscribersQueryGeneratorTx = getSubscribersQueryGeneratorTx;
module.exports.prepareCampaignMessages = prepareCampaignMessages;
module.exports.start = start;
module.exports.stop = stop;

11
server/models/fields.js
View file

@ -20,8 +20,8 @@ const {ListActivityType} = require('../../shared/activity-log');
const activityLog = require('../lib/activity-log');
const allowedKeysCreate = new Set(['name', 'key', 'default_value', 'type', 'group', 'settings']);
const allowedKeysUpdate = new Set(['name', 'key', 'default_value', 'group', 'settings']);
const allowedKeysCreate = new Set(['name', 'help', 'key', 'default_value', 'type', 'group', 'settings']);
const allowedKeysUpdate = new Set(['name', 'help', 'key', 'default_value', 'group', 'settings']);
const hashKeys = allowedKeysCreate;
const fieldTypes = {};
@ -304,7 +304,7 @@ async function getById(context, listId, id) {
}
async function listTx(tx, listId) {
return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'key', 'column', 'settings', 'group', 'default_value', 'order_list', 'order_subscribe', 'order_manage']).orderBy(knex.raw('-order_list'), 'desc').orderBy('id', 'asc');
return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'help', 'key', 'column', 'settings', 'group', 'default_value', 'order_list', 'order_subscribe', 'order_manage']).orderBy(knex.raw('-order_list'), 'desc').orderBy('id', 'asc');
}
async function list(context, listId) {
@ -661,10 +661,11 @@ function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes group
const entry = {
name: fld.name,
key: fld.key,
help: fld.help,
field: fld,
[type.getHbsType(fld)]: true,
order_subscribe: fld.order_subscribe,
order_manage: fld.order_manage,
order_manage: fld.order_manage
};
if (!type.grouped && !type.enumerated) {
@ -688,6 +689,7 @@ function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes group
options.push({
key: opt.key,
name: opt.name,
help: opt.help,
value: isEnabled
});
}
@ -702,6 +704,7 @@ function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes group
options.push({
key: opt.key,
name: opt.label,
help: opt.help,
value: value === opt.key
});
}

19
server/models/subscriptions.js
View file

@ -11,10 +11,9 @@ const fields = require('./fields');
const { SubscriptionSource, SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
const { CampaignMessageStatus } = require('../../shared/campaigns');
const segments = require('./segments');
const { enforce, filterObject } = require('../lib/helpers');
const { enforce, filterObject, hashEmail, normalizeEmail } = require('../lib/helpers');
const moment = require('moment');
const { formatDate, formatBirthday } = require('../../shared/date');
const crypto = require('crypto');
const campaigns = require('./campaigns');
const lists = require('./lists');
@ -85,7 +84,6 @@ fieldTypes.option = {
};
function getSubscriptionTableName(listId) {
return `subscription__${listId}`;
}
@ -232,7 +230,12 @@ async function getById(context, listId, id, grouped = true) {
}
async function getByEmail(context, listId, email, grouped = true) {
return await _getBy(context, listId, 'email', email, grouped);
const result = await _getBy(context, listId, 'hash_email', hashEmail(email), grouped);
if (result.email === null) {
throw new interoperableErrors.NotFoundError('Subscription not found in this list');
}
enforce(normalizeEmail(email) === normalizeEmail(result.email));
return result;
}
async function getByCid(context, listId, cid, grouped = true) {
@ -486,7 +489,7 @@ async function serverValidate(context, listId, data) {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
if (data.email) {
const existingKeyQuery = tx(getSubscriptionTableName(listId)).where('email', data.email);
const existingKeyQuery = tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(data.email)).whereNotNull('email');
if (data.id) {
existingKeyQuery.whereNot('id', data.id);
@ -539,10 +542,6 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
}
}
function hashEmail(email) {
return crypto.createHash('sha512').update(email).digest("base64");
}
function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
if ('email' in subscription) {
subscription.hash_email = hashEmail(subscription.email);
@ -865,7 +864,7 @@ async function getListsWithEmail(context, email) {
for (const list of lsts) {
await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
const entity = await tx(getSubscriptionTableName(list.id)).where('email', email).first();
const entity = await tx(getSubscriptionTableName(list.id)).where('hash_email', hashEmail(email)).whereNotNull('email').first();
if (entity) {
result.push(list);
}

39
server/services/sender-master.js
View file

@ -5,7 +5,7 @@ const fork = require('../lib/fork').fork;
const log = require('../lib/log');
const path = require('path');
const knex = require('../lib/knex');
const {CampaignStatus, CampaignType} = require('../../shared/campaigns');
const {CampaignStatus, CampaignType, CampaignMessageStatus} = require('../../shared/campaigns');
const campaigns = require('../models/campaigns');
const builtinZoneMta = require('../lib/builtin-zone-mta');
const {CampaignActivityType} = require('../../shared/activity-log');
@ -62,10 +62,10 @@ const workerBatchSize = 10;
const sendConfigurationIdByCampaignId = new Map(); // campaignId -> sendConfigurationId
const sendConfigurationStatuses = new Map(); // sendConfigurationId -> {retryCount, postponeTill}
const sendConfigurationMessageQueue = new Map(); // sendConfigurationId -> [{queuedMessage}]
const campaignMessageQueue = new Map(); // campaignId -> [{listId, email}]
const sendConfigurationMessageQueue = new Map(); // sendConfigurationId -> [queuedMessage]
const campaignMessageQueue = new Map(); // campaignId -> [campaignMessage]
const workAssignment = new Map(); // workerId -> { type: WorkAssignmentType.CAMPAIGN, campaignId, messages: [{listId, email} } / { type: WorkAssignmentType.QUEUED, sendConfigurationId, messages: [{queuedMessage}] }
const workAssignment = new Map(); // workerId -> { type: WorkAssignmentType.CAMPAIGN, campaignId, messages: [campaignMessage] / { type: WorkAssignmentType.QUEUED, sendConfigurationId, messages: [queuedMessage] }
const WorkAssignmentType = {
CAMPAIGN: 0,
@ -330,7 +330,10 @@ async function processCampaign(campaignId) {
}
}
try {
await campaigns.prepareCampaignMessages(campaignId);
while (true) {
const cpg = await knex('campaigns').where('id', campaignId).first();
@ -350,12 +353,6 @@ async function processCampaign(campaignId) {
return await finish(true, CampaignStatus.SCHEDULED);
}
let qryGen;
await knex.transaction(async tx => {
qryGen = await campaigns.getSubscribersQueryGeneratorTx(tx, campaignId);
});
if (qryGen) {
let messagesInProcessing = [...msgQueue];
for (const wa of workAssignment.values()) {
if (wa.type === WorkAssignmentType.CAMPAIGN && wa.campaignId === campaignId) {
@ -363,11 +360,10 @@ async function processCampaign(campaignId) {
}
}
const qry = qryGen(knex)
.whereNotIn('pending_subscriptions.email', messagesInProcessing.map(x => x.email))
.select(['pending_subscriptions.email', 'campaign_lists.list'])
const subs = await knex('campaign_messages')
.where({status: CampaignMessageStatus.SCHEDULED})
.whereNotIn('hash_email', messagesInProcessing.map(x => x.hash_email))
.limit(retrieveBatchSize);
const subs = await qry;
if (subs.length === 0) {
if (isCompleted()) {
@ -383,10 +379,7 @@ async function processCampaign(campaignId) {
}
for (const sub of subs) {
msgQueue.push({
listId: sub.list,
email: sub.email
});
msgQueue.push(sub);
}
notifier.notify('workAvailable');
@ -394,10 +387,6 @@ async function processCampaign(campaignId) {
while (msgQueue.length > 0) {
await notifier.waitFor(`campaignMessageQueueEmpty:${campaignId}`);
}
} else {
return await finish(false, CampaignStatus.FINISHED);
}
}
} catch (err) {
log.error('Senders', `Sending campaign ${campaignId} failed with error: ${err.message}`);
@ -526,7 +515,7 @@ async function processQueuedBySendConfiguration(sendConfigurationId) {
}
}
const messageIdsInProcessing = messagesInProcessing.map(x => x.queuedMessage.id);
const messageIdsInProcessing = messagesInProcessing.map(x => x.id);
const rows = await knex('queued')
.orderByRaw(`FIELD(type, ${MessageType.TRIGGERED}, ${MessageType.API_TRANSACTIONAL}, ${MessageType.TEST}, ${MessageType.SUBSCRIPTION}) DESC, id ASC`) // This orders messages in the following order MessageType.SUBSCRIPTION, MessageType.TEST, MessageType.API_TRANSACTIONAL and MessageType.TRIGGERED
@ -561,9 +550,7 @@ async function processQueuedBySendConfiguration(sendConfigurationId) {
} else {
row.data = JSON.parse(row.data);
msgQueue.push({
queuedMessage: row
});
msgQueue.push(row);
}
}

13
server/services/sender-worker.js
View file

@ -22,20 +22,20 @@ async function processCampaignMessages(campaignId, messages) {
let withErrors = false;
for (const msg of messages) {
for (const campaignMessage of messages) {
try {
await cs.sendRegularCampaignMessage(msg.listId, msg.email);
await cs.sendRegularCampaignMessage(campaignMessage);
log.verbose('Senders', 'Message sent and status updated for %s:%s', msg.listId, msg.email);
log.verbose('Senders', 'Message sent and status updated for %s:%s', campaignMessage.list, campaignMessage.subscription);
} catch (err) {
if (err instanceof mailers.SendConfigurationError) {
log.error('Senders', `Sending message to ${msg.listId}:${msg.email} failed with error: ${err.message}. Will retry the message if within retention interval.`);
log.error('Senders', `Sending message to ${campaignMessage.list}:${campaignMessage.subscription} failed with error: ${err.message}. Will retry the message if within retention interval.`);
withErrors = true;
break;
} else {
log.error('Senders', `Sending message to ${msg.listId}:${msg.email} failed with error: ${err.message}.`);
log.error('Senders', `Sending message to ${campaignMessage.list}:${campaignMessage.subscription} failed with error: ${err.message}.`);
log.verbose(err.stack);
}
}
@ -56,8 +56,7 @@ async function processQueuedMessages(sendConfigurationId, messages) {
let withErrors = false;
for (const msg of messages) {
const queuedMessage = msg.queuedMessage;
for (const queuedMessage of messages) {
const msgData = queuedMessage.data;
let target = '';

6
server/setup/knex/migrations/20170506102634_v1_to_v2.js
View file

@ -6,10 +6,9 @@ const {TagLanguages} = require('../../../../shared/templates');
const {getGlobalNamespaceId} = require('../../../../shared/namespaces');
const {getAdminId} = require('../../../../shared/users');
const { MailerType, ZoneMTAType, getSystemSendConfigurationId, getSystemSendConfigurationCid } = require('../../../../shared/send-configurations');
const { enforce } = require('../../../lib/helpers');
const { enforce, hashEmail} = require('../../../lib/helpers');
const { EntityVals: TriggerEntityVals, EventVals: TriggerEventVals } = require('../../../../shared/triggers');
const { SubscriptionSource } = require('../../../../shared/lists');
const crypto = require('crypto');
const {DOMParser, XMLSerializer} = require('xmldom');
const log = require('../../../lib/log');
@ -271,7 +270,7 @@ async function migrateSubscriptions(knex) {
if (rows.length > 0) {
for await (const subscription of rows) {
subscription.hash_email = crypto.createHash('sha512').update(subscription.email).digest("base64");
subscription.hash_email = hashEmail(subscription.email);
subscription.source_email = subscription.imported ? SubscriptionSource.IMPORTED_V1 : SubscriptionSource.NOT_IMPORTED_V1;
for (const field of fields) {
if (field.column != null) {
@ -417,6 +416,7 @@ async function migrateCustomFields(knex) {
});
await knex.schema.table('custom_fields', table => {
table.renameColumn('description', 'help');
table.dropColumn('visible');
});

17
server/setup/knex/migrations/20190722110000_hash_email.js Normal file
View file

@ -0,0 +1,17 @@
exports.up = (knex, Promise) => (async() => {
await knex.schema.raw('ALTER TABLE `campaign_messages` ADD `hash_email` char(88) CHARACTER SET ascii');
await knex.schema.raw('ALTER TABLE `campaign_messages` ADD UNIQUE KEY `campaign_hash_email` (`campaign`, `hash_email`)');
await knex.schema.raw('ALTER TABLE `campaign_messages` DROP KEY `created`');
await knex.schema.raw('ALTER TABLE `campaign_links` DROP KEY `created_index`');
const lists = await knex('lists');
for (const list of lists) {
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` MODIFY `hash_email` char(88) CHARACTER SET ascii');
await knex.raw('update `campaign_messages` inner join `subscription__' + list.id + '` on `campaign_messages`.`list`=' + list.id + ' and `campaign_messages`.`subscription`=`subscription__' + list.id + '`.`id` set `campaign_messages`.`hash_email`=`subscription__' + list.id + '`.`hash_email`');
}
await knex('campaign_messages').whereNull('hash_email').del();
})();
exports.down = (knex, Promise) => (async() => {
})();

7
server/setup/knex/migrations/20190722150000_ensure_help_column_in_custom_fields.js Normal file
View file

@ -0,0 +1,7 @@
exports.up = (knex, Promise) => (async() => {
// This is to provide upgrade path to stable to those that already have beta installed.
await knex.schema.raw('ALTER TABLE `custom_fields` ADD COLUMN `help` text AFTER `name`');
})();
exports.down = (knex, Promise) => (async() => {
})();

11
server/setup/sql/upgrade-00034.sql Normal file
View file

@ -0,0 +1,11 @@
# Header section
# Define incrementing schema version number
SET @schema_version = '34';
# Add template field for group elements
ALTER TABLE `custom_fields` ADD COLUMN `description` text AFTER `name`;
# Footer section
LOCK TABLES `settings` WRITE;
INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
UNLOCK TABLES;

21
server/views/subscription/partials/subscription-custom-fields.hbs
View file

@ -11,6 +11,7 @@
{{else}}
<input type="email" name="EMAIL" id="email" placeholder="" value="{{../email}}" required>
{{/if}}
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -18,6 +19,7 @@
<div class="form-group text {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="text" name="{{key}}" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -25,6 +27,7 @@
<div class="form-group number {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="number" name="{{key}}" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -32,6 +35,7 @@
<div class="form-group url {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="url" name="{{key}}" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -39,6 +43,7 @@
<div class="form-group longtext {{key}}">
<label for="{{key}}">{{name}}</label>
<textarea rows="3" name="{{key}}">{{value}}</textarea>
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -46,6 +51,7 @@
<div class="form-group json {{key}}">
<label for="{{key}}">{{name}}</label>
<textarea class="gpg-text" rows="3" name="{{key}}" placeholder="{&quot;data&quot;:&quot;value&quot;}">{{value}}</textarea>
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -56,6 +62,7 @@
<input type="checkbox" name="{{key}}" value="1" {{#if value}} checked {{/if}}> {{field.settings.checkedLabel}}
</label>
<input type="hidden" value="0" name="{{key}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -66,9 +73,8 @@
<button class="btn-download-pubkey" type="submit" form="download-pubkey">{{#translate}}downloadSignatureVerificationKey{{/translate}}</button>
{{/if}}
<textarea class="form-control gpg-text" rows="4" name="{{key}}" placeholder="{{#translate}}beginsWithAnd#39BeginPgpPublicKeyBloc{{/translate}}">{{value}}</textarea>
<span class="help-block">
{{#translate}}insertYourGpgPublicKeyHereToEncrypt{{/translate}}
</span>
<small class="form-text text-muted">{{#translate}}insertYourGpgPublicKeyHereToEncrypt{{/translate}}</small>
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -76,6 +82,7 @@
<div class="form-group date fm-date-us {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="text" name="{{key}}" placeholder="MM/DD/YYYY" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -83,6 +90,7 @@
<div class="form-group date fm-date-eur {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="text" name="{{key}}" placeholder="DD/MM/YYYY" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -90,6 +98,7 @@
<div class="form-group date fm-birthday-us {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="text" name="{{key}}" placeholder="MM/DD" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -97,6 +106,7 @@
<div class="form-group date fm-birthday-eur {{key}}">
<label for="{{key}}">{{name}}</label>
<input type="text" name="{{key}}" placeholder="DD/MM" value="{{value}}">
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -111,6 +121,7 @@
<option value="{{key}}" {{#if value}} selected {{/if}}>{{name}}</option>
{{/each}}
</select>
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -122,6 +133,7 @@
<input type="radio" name="{{../key}}" value="{{key}}" {{#if value}} checked {{/if}}> {{name}}
</label>
{{/each}}
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -133,6 +145,7 @@
<label class="label-checkbox">
<input type="checkbox" name="{{key}}" value="1" {{#if value}} checked {{/if}}> {{name}}
</label>
<small class="option-help-block form-text text-muted">{{help}}</small>
{{/each}}
</div>
{{/if}}
@ -148,6 +161,7 @@
<option value="{{key}}" {{#if value}} selected {{/if}}>{{name}}</option>
{{/each}}
</select>
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}
@ -159,6 +173,7 @@
<input type="radio" name="{{../key}}" value="{{key}}" {{#if value}} checked {{/if}}> {{name}}
</label>
{{/each}}
<small class="form-text text-muted">{{help}}</small>
</div>
{{/if}}

2
shared/campaigns.js
View file

@ -51,7 +51,7 @@ const CampaignMessageStatus = {
UNSUBSCRIBED: 2,
BOUNCED: 3,
COMPLAINED: 4,
SENDING: 5,
SCHEDULED: 5,
MAX: 5
};