public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fixes in subscriptions. It now passes the tests.

Browse source 
API tests still don't work.
This commit is contained in:
Tomas Bures 2018-01-28 23:59:05 +01:00
parent e9165838dc
commit 47b8d80c22
16 changed files with 2649 additions and 975 deletions
Download patch file Download diff file Expand all files Collapse all files

36
models/blacklist.js
View file

@ -16,45 +16,15 @@ async function listDTAjax(context, params) {
);
}
/*
module.exports.get = (start, limit, search, callback) => {
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
search = '%' + search + '%';
connection.query('SELECT SQL_CALC_FOUND_ROWS `email` FROM blacklist WHERE `email` LIKE ? ORDER BY `email` LIMIT ? OFFSET ?', [search, limit, start], (err, rows) => {
if (err) {
return callback(err);
}
connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
connection.release();
if (err) {
return callback(err);
}
let emails = [];
rows.forEach(email => {
emails.push(email.email);
});
return callback(null, emails, total && total[0] && total[0].total);
});
});
});
};
*/
async function search(context, start, limit, search) {
async function search(context, offset, limit, search) {
return await knex.transaction(async tx => {
shares.enforceGlobalPermission(context, 'manageBlacklist');
search = '%' + search + '%';
const count = await tx('blacklist').where('email', 'like', search).count();
// FIXME - the count won't likely work;
console.log(count);
const count = await tx('blacklist').where('email', 'like', search).count('* as count').first().count;
const rows = await tx('blacklist').where('email', 'like', search).offset(start).limit(limit);
const rows = await tx('blacklist').where('email', 'like', search).offset(offset).limit(limit);
return {
emails: rows.map(row => row.email),

250
models/fields.js
View file

@ -10,6 +10,10 @@ const shares = require('./shares');
const validators = require('../shared/validators');
const shortid = require('shortid');
const segments = require('./segments');
const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../shared/date');
const { getFieldKey } = require('../shared/lists');
const { cleanupFromPost } = require('../lib/helpers');
const allowedKeysCreate = new Set(['name', 'key', 'default_value', 'type', 'group', 'settings']);
const allowedKeysUpdate = new Set(['name', 'key', 'default_value', 'group', 'settings']);
@ -22,72 +26,133 @@ const Cardinality = {
MULTIPLE: 1
};
fieldTypes.text = fieldTypes.website = {
validate: entity => {},
fieldTypes.text = {
validate: field => {},
addColumn: (table, name) => table.string(name),
indexed: true,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeText',
forHbs: (field, value) => value,
parsePostValue: (field, value) => value
};
fieldTypes.longtext = fieldTypes.gpg = {
validate: entity => {},
fieldTypes.website = {
validate: field => {},
addColumn: (table, name) => table.string(name),
indexed: true,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeWebsite',
forHbs: (field, value) => value,
parsePostValue: (field, value) => value
};
fieldTypes.longtext = {
validate: field => {},
addColumn: (table, name) => table.text(name),
indexed: false,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeLongtext',
forHbs: (field, value) => value,
parsePostValue: (field, value) => value
};
fieldTypes.gpg = {
validate: field => {},
addColumn: (table, name) => table.text(name),
indexed: false,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeGpg',
forHbs: (field, value) => value,
parsePostValue: (field, value) => value
};
fieldTypes.json = {
validate: entity => {},
validate: field => {},
addColumn: (table, name) => table.json(name),
indexed: false,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeJson',
forHbs: (field, value) => value,
parsePostValue: (field, value) => value
};
fieldTypes.number = {
validate: entity => {},
validate: field => {},
addColumn: (table, name) => table.integer(name),
indexed: true,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeNumber',
forHbs: (field, value) => value,
parsePostValue: (field, value) => Number(value)
};
fieldTypes['checkbox-grouped'] = {
validate: entity => {},
validate: field => {},
indexed: true,
grouped: true,
enumerated: false,
cardinality: Cardinality.MULTIPLE
cardinality: Cardinality.MULTIPLE,
getHbsType: field => 'typeCheckboxGrouped'
};
fieldTypes['radio-grouped'] = fieldTypes['dropdown-grouped'] = {
validate: entity => {},
fieldTypes['radio-grouped'] = {
validate: field => {},
indexed: true,
grouped: true,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeRadioGrouped'
};
fieldTypes['radio-enum'] = fieldTypes['dropdown-enum'] = {
validate: entity => {
enforce(entity.settings.options, 'Options missing in settings');
enforce(entity.default_value === null || entity.settings.options.find(x => x.key === entity.default_value), 'Default value not present in options');
fieldTypes['dropdown-grouped'] = {
validate: field => {},
indexed: true,
grouped: true,
enumerated: false,
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeDropdownGrouped'
};
fieldTypes['radio-enum'] = {
validate: field => {
enforce(field.settings.options, 'Options missing in settings');
enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
},
addColumn: (table, name) => table.string(name),
indexed: true,
grouped: false,
enumerated: true,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeRadioEnum'
};
fieldTypes['dropdown-enum'] = {
validate: field => {
enforce(field.settings.options, 'Options missing in settings');
enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
},
addColumn: (table, name) => table.string(name),
indexed: true,
grouped: false,
enumerated: true,
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeDropdownEnum'
};
fieldTypes.option = {
validate: entity => {},
validate: field => {},
addColumn: (table, name) => table.boolean(name),
indexed: true,
grouped: false,
@ -95,15 +160,32 @@ fieldTypes.option = {
cardinality: Cardinality.SINGLE
};
fieldTypes['date'] = fieldTypes['birthday'] = {
validate: entity => {
enforce(['eur', 'us'].includes(entity.settings.dateFormat), 'Date format incorrect');
fieldTypes['date'] = {
validate: field => {
enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
},
addColumn: (table, name) => table.dateTime(name),
indexed: true,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeDate' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
forHbs: (field, value) => formatDate(field.settings.dateFormat, value),
parsePostValue: (field, value) => parseDate(field.settings.dateFormat, value)
};
fieldTypes['birthday'] = {
validate: field => {
enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
},
addColumn: (table, name) => table.dateTime(name),
indexed: true,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE,
getHbsType: field => 'typeBirthday' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
forHbs: (field, value) => formatBirthday(field.settings.dateFormat, value),
parsePostValue: (field, value) => parseBirthday(field.settings.dateFormat, value)
};
const groupedTypes = Object.keys(fieldTypes).filter(key => fieldTypes[key].grouped);
@ -153,7 +235,7 @@ async function listTx(tx, listId) {
async function list(context, listId) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageFields', 'manageSegments']);
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageFields', 'manageSegments', 'manageSubscriptions']);
return await listTx(tx, listId);
});
}
@ -474,32 +556,127 @@ async function removeAllByListIdTx(tx, context, listId) {
}
}
async function getRow(context, listId, subscription) {
// Returns an array that can be used for rendering by Handlebars
async function forHbs(context, listId, subscription) { // assumes grouped subscription
const customFields = [{
name: 'Email Address',
column: 'email',
key: 'EMAIL',
typeSubscriptionEmail: true,
value: subscription ? subscription.email : '',
order_subscribe: -1,
order_manage: -1
}];
const flds = await list(context, listId);
const flds = await listGrouped(context, listId);
for (const fld of flds) {
if (fld.column) {
customFields.push({
name: fld.name,
column: fld.column,
['type' + fld.type.replace(/(?:^|-)([a-z])/g, (m, c) => c.toUpperCase())]: true,
value: subscription ? subscription[fld.column] : ''
});
const type = fieldTypes[fld.type];
const fldKey = getFieldKey(fld);
const entry = {
name: fld.name,
key: fld.key,
[type.getHbsType(fld)]: true,
order_subscribe: fld.order_subscribe,
order_manage: fld.order_manage,
};
if (!type.grouped && !type.enumerated) {
entry.value = subscription ? type.forHbs(fld, subscription[fldKey]) : '';
} else if (type.grouped) {
const options = [];
const value = subscription ? subscription[fldKey] : (type.cardinality === Cardinality.SINGLE ? null : []);
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
let isEnabled;
if (type.cardinality === Cardinality.SINGLE) {
isEnabled = value === opt.column;
} else {
isEnabled = value.includes(opt.column);
}
options.push({
key: opt.key,
name: opt.name,
value: isEnabled
});
}
entry.options = options;
} else if (type.enumerated) {
const options = [];
const value = subscription ? subscription[fldKey] : null;
for (const opt of fld.settings.options) {
options.push({
key: opt.key,
name: opt.label,
value: value === opt.key
});
}
entry.options = options;
}
customFields.push(entry);
}
return customFields;
}
async function fromPost(context, listId, data) { // assumes grouped subscription
const flds = await listGrouped(context, listId);
const subscription = {};
for (const fld of flds) {
const type = fieldTypes[fld.type];
const fldKey = getFieldKey(fld);
let value = null;
if (!type.grouped && !type.enumerated) {
value = type.parsePostValue(fld, cleanupFromPost(data[fld.key]));
} else if (type.grouped) {
if (type.cardinality === Cardinality.SINGLE) {
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
if (data[fld.key] === opt.key) {
value = opt.column
}
}
} else {
value = [];
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
if (data[opt.key]) {
value.push(opt.column);
}
}
}
} else if (type.enumerated) {
value = data[fld.key];
}
subscription[fldKey] = value;
}
return subscription;
}
// This is to handle circular dependency with segments.js
Object.assign(module.exports, {
Cardinality,
@ -518,5 +695,6 @@ Object.assign(module.exports, {
remove,
removeAllByListIdTx,
serverValidate,
getRow
forHbs,
fromPost
});

3
models/shares.js
View file

@ -8,6 +8,9 @@ const dtHelpers = require('../lib/dt-helpers');
const permissions = require('../lib/permissions');
const interoperableErrors = require('../shared/interoperable-errors');
// TODO: This would really benefit from some permission cache connected to rebuildPermissions
// A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions
async function listByEntityDTAjax(context, entityTypeId, entityId, params) {
return await knex.transaction(async (tx) => {

179
models/subscriptions.js
View file

@ -31,17 +31,7 @@ function getOptionsMap(groupedField) {
return result;
}
fieldTypes.text = fieldTypes.website = fieldTypes.longtext = fieldTypes.gpg = {
afterJSON: (groupedField, entity) => {},
listRender: (groupedField, value) => value
};
fieldTypes.number = {
afterJSON: (groupedField, entity) => {},
listRender: (groupedField, value) => Number(value)
};
fieldTypes.json = {
fieldTypes.text = fieldTypes.website = fieldTypes.longtext = fieldTypes.gpg = fieldTypes.number = fieldTypes.json = {
afterJSON: (groupedField, entity) => {},
listRender: (groupedField, value) => value
};
@ -207,26 +197,16 @@ async function hashByList(listId, entity) {
}
async function _getStatusBy(context, listId, key, value) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
const entity = await tx(getSubscriptionTableName(listId)).where(key, value).select(['status']).first();
if (!entity) {
throw new interoperableErrors.NotFoundError();
}
return entity.status;
});
}
async function _getBy(context, listId, key, value, grouped) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
const entity = await tx(getSubscriptionTableName(listId)).where(key, value).first();
if (!entity) {
throw new interoperableErrors.NotFoundError('Subscription not found in this list');
}
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
if (grouped) {
@ -355,19 +335,36 @@ async function listDTAjax(context, listId, segmentId, params) {
});
}
async function list(context, listId) {
async function list(context, listId, grouped = true, offset, limit) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
const entities = await tx(getSubscriptionTableName(listId));
const count = await tx(getSubscriptionTableName(listId)).count('* as count').first().count;
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
const entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc');
for (const entity of entities) {
groupSubscription(groupedFieldsMap, entity);
if (Number.isInteger(offset)) {
entitiesQry.offset(offset);
}
return entities;
if (Number.isInteger(limit)) {
entitiesQry.limit(limit);
}
const entities = await entitiesQry;
if (grouped) {
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
for (const entity of entities) {
groupSubscription(groupedFieldsMap, entity);
}
}
return {
subscriptions: entities,
total: count
};
});
}
@ -395,7 +392,6 @@ async function serverValidate(context, listId, data) {
async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) {
enforce(entity.email, 'Email must be set');
enforce(entity.status > 0 && entity.status < SubscriptionStatus.MAX, 'Subscription status is invalid');
const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('email', entity.email);
@ -405,14 +401,16 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
const existingWithKey = await existingWithKeyQuery.first();
if (existingWithKey) {
if (meta && meta.replaceOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED) {
meta.updateNeeded = true;
meta.update = true;
meta.existing = existingWithKey;
} else {
throw new interoperableErrors.DuplicitEmailError();
}
}
enforce(entity.status >= 0 && entity.status < SubscriptionStatus.MAX, 'Invalid status');
if ((isCreate && !(meta && meta.update)) || 'status' in entity) {
enforce(entity.status >= 0 && entity.status < SubscriptionStatus.MAX, 'Invalid status');
}
for (const key in groupedFieldsMap) {
const fld = groupedFieldsMap[key];
@ -422,22 +420,25 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
}
async function _update(tx, listId, existing, filteredEntity) {
if (existing.status !== filteredEntity.status) {
filteredEntity.status_change = new Date();
if ('status' in filteredEntity) {
if (existing.status !== filteredEntity.status) {
filteredEntity.status_change = new Date();
}
}
await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
if ('status' in filteredEntity) {
let countIncrement = 0;
if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
countIncrement = -1;
} else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
countIncrement = 1;
}
let countIncrement = 0;
if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
countIncrement = -1;
} else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
countIncrement = 1;
}
if (countIncrement) {
await tx('lists').where('id', listId).increment('subscribers', countIncrement);
if (countIncrement) {
await tx('lists').where('id', listId).increment('subscribers', countIncrement);
}
}
}
@ -452,6 +453,11 @@ async function _create(tx, listId, filteredEntity) {
return id;
}
/*
Adds a new subscription. Returns error if a subscription with the same email address is already present and is not unsubscribed.
If it is unsubscribed, the existing subscription is changed based on the provided data.
If meta.partial is true, it updates even an active subscription.
*/
async function create(context, listId, entity, meta /* meta is provided when called from /confirm/subscribe/:cid */) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
@ -470,7 +476,7 @@ async function create(context, listId, entity, meta /* meta is provided when cal
filteredEntity.opt_in_country = meta && meta.country;
filteredEntity.imported = meta && !!meta.imported;
if (meta && meta.updateNeeded) {
if (meta && meta.update) {
await _update(tx, listId, meta.existing, filteredEntity);
meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
return meta.existing.id;
@ -515,10 +521,9 @@ async function updateWithConsistencyCheck(context, listId, entity) {
});
}
async function removeTx(tx, context, listId, id) {
async function _removeAndGetTx(tx, context, listId, existing) {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
const existing = await tx(getSubscriptionTableName(listId)).where('id', id).first();
if (!existing) {
throw new interoperableErrors.NotFoundError();
}
@ -532,10 +537,19 @@ async function removeTx(tx, context, listId, id) {
async function remove(context, listId, id) {
await knex.transaction(async tx => {
await removeTx(tx, context, listId, id);
const existing = await tx(getSubscriptionTableName(listId)).where('id', id).first();
await _removeAndGetTx(tx, context, listId, existing);
});
}
async function removeByEmailAndGet(context, listId, email) {
return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first();
return await _removeAndGetTx(tx, context, listId, existing);
});
}
async function _unsubscribeAndGetTx(tx, context, listId, existingSubscription, campaignCid) {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
@ -585,6 +599,13 @@ async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaign
});
}
async function unsubscribeByEmailAndGet(context, listId, email) {
return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first();
return _unsubscribeAndGetTx(tx, context, listId, existing);
});
}
async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
@ -594,39 +615,60 @@ async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
throw new interoperableErrors.NotFoundError();
}
await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
email: emailNew
});
if (existing.email !== emailNew) {
await tx(getSubscriptionTableName(listId)).where('email', emailNew).del();
await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
email: emailNew
});
existing.email = emailNew;
}
existing.email = emailNew;
return existing;
});
}
async function updateManagedUngrouped(context, listId, entity) {
async function updateManaged(context, listId, cid, entity) {
await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
const existing = await tx(getSubscriptionTableName(listId)).where('cid', entity.cid).first();
if (!existing || existing.status !== SubscriptionStatus.SUBSCRIBED) {
throw new interoperableErrors.NotFoundError();
}
const flds = await fields.listTx(tx, listId);
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
const update = {};
for (const key in groupedFieldsMap) {
const fld = groupedFieldsMap[key];
for (const fld of flds) {
if (fld.order_manage) {
if (!fld.group) { // fieldTypes is primarily meant only for groupedFields, so we don't try it for fields that would be grouped (i.e. option), because there is nothing to be done for them anyway
fieldTypes[fld.type].afterJSON(fld, entity);
}
update[key] = entity[key];
}
update[fld.column] = entity[fld.column];
fieldTypes[fld.type].afterJSON(fld, update);
}
ungroupSubscription(groupedFieldsMap, update);
await tx(getSubscriptionTableName(listId)).where('cid', cid).update(update);
});
}
async function getListsWithEmail(context, email) {
// FIXME - this methods is rather suboptimal if there are many lists. It quite needs permission caching in shares.js
return await knex.transaction(async tx => {
const lists = await tx('lists').select(['id', 'name']);
const result = [];
for (const list of lists) {
await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
const entity = await tx(getSubscriptionTableName(list.id)).where('email', email).first();
if (entity) {
result.push(list);
}
}
await tx(getSubscriptionTableName(listId)).where('cid', entity.cid).update(update);
return result;
});
}
@ -641,8 +683,11 @@ module.exports = {
create,
updateWithConsistencyCheck,
remove,
removeByEmailAndGet,
unsubscribeByCidAndGet,
unsubscribeByIdAndGet,
unsubscribeByEmailAndGet,
updateAddressAndGet,
updateManagedUngrouped
updateManaged,
getListsWithEmail
};