WiP on segments

This commit is contained in:
Tomas Bures 2017-08-19 15:12:22 +02:00
parent 6cc34136f5
commit f3ff89c536
21 changed files with 945 additions and 352 deletions

View file

@ -113,10 +113,14 @@ async function getById(context, listId, id) {
});
}
async function listTx(tx, listId) {
return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'key', 'column', 'order_list', 'order_subscribe', 'order_manage']).orderBy('id', 'asc');
}
async function list(context, listId) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'key', 'column', 'order_list', 'order_subscribe', 'order_manage']).orderBy('id', 'asc');
return await listTx(tx, listId);
});
}
@ -125,72 +129,76 @@ async function listByOrderListTx(tx, listId, extraColumns = []) {
}
async function listDTAjax(context, listId, params) {
return await dtHelpers.ajaxListWithPermissions(
context,
[{ entityTypeId: 'list', requiredOperations: ['manageFields'] }],
params,
builder => builder
.from('custom_fields')
.innerJoin('lists', 'custom_fields.list', 'lists.id')
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
// This self join is to provide 'option' fields a reference to their parent grouped field. If the field is not an option, it refers to itself
// All this is to show options always below their group parent
.innerJoin('custom_fields AS parent_fields', function() {
this.on(function() {
this.on('custom_fields.type', '=', knex.raw('?', ['option']))
.on('custom_fields.group', '=', 'parent_fields.id');
}).orOn(function() {
this.on('custom_fields.type', '<>', knex.raw('?', ['option']))
.on('custom_fields.id', '=', 'parent_fields.id');
});
})
.where('custom_fields.list', listId),
[ 'custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list' ],
{
orderByBuilder: (builder, orderColumn, orderDir) => {
// We use here parent_fields to keep options always below their parent group
if (orderColumn === 'custom_fields.order_list') {
builder
.orderBy(knex.raw('-parent_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
.orderBy('parent_fields.name', orderDir)
.orderBy(knex.raw('custom_fields.type = "option"'), 'asc')
} else {
const parentColumn = orderColumn.replace(/^custom_fields/, 'parent_fields');
builder
.orderBy(parentColumn, orderDir)
.orderBy('parent_fields.name', orderDir)
.orderBy(knex.raw('custom_fields.type = "option"'), 'asc');
return await dtHelpers.ajaxListTx(
tx,
params,
builder => builder
.from('custom_fields')
// This self join is to provide 'option' fields a reference to their parent grouped field. If the field is not an option, it refers to itself
// All this is to show options always below their group parent
.innerJoin('custom_fields AS parent_fields', function() {
this.on(function() {
this.on('custom_fields.type', '=', knex.raw('?', ['option']))
.on('custom_fields.group', '=', 'parent_fields.id');
}).orOn(function() {
this.on('custom_fields.type', '<>', knex.raw('?', ['option']))
.on('custom_fields.id', '=', 'parent_fields.id');
});
})
.where('custom_fields.list', listId),
[ 'custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list' ],
{
orderByBuilder: (builder, orderColumn, orderDir) => {
// We use here parent_fields to keep options always below their parent group
if (orderColumn === 'custom_fields.order_list') {
builder
.orderBy(knex.raw('-parent_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
.orderBy('parent_fields.name', orderDir)
.orderBy(knex.raw('custom_fields.type = "option"'), 'asc')
} else {
const parentColumn = orderColumn.replace(/^custom_fields/, 'parent_fields');
builder
.orderBy(parentColumn, orderDir)
.orderBy('parent_fields.name', orderDir)
.orderBy(knex.raw('custom_fields.type = "option"'), 'asc');
}
}
}
}
);
);
});
}
async function listGroupedDTAjax(context, listId, params) {
return await dtHelpers.ajaxListWithPermissions(
context,
[{ entityTypeId: 'list', requiredOperations: ['manageFields'] }],
params,
builder => builder
.from('custom_fields')
.innerJoin('lists', 'custom_fields.list', 'lists.id')
.where('custom_fields.list', listId)
.whereIn('custom_fields.type', groupedTypes),
[ 'custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list' ],
{
orderByBuilder: (builder, orderColumn, orderDir) => {
if (orderColumn === 'custom_fields.order_list') {
builder
.orderBy(knex.raw('-custom_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
.orderBy('custom_fields.name', orderDir);
} else {
builder
.orderBy(orderColumn, orderDir)
.orderBy('custom_fields.name', orderDir);
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
return await dtHelpers.ajaxListTx(
tx,
params,
builder => builder
.from('custom_fields')
.where('custom_fields.list', listId)
.whereIn('custom_fields.type', groupedTypes),
['custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list'],
{
orderByBuilder: (builder, orderColumn, orderDir) => {
if (orderColumn === 'custom_fields.order_list') {
builder
.orderBy(knex.raw('-custom_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
.orderBy('custom_fields.name', orderDir);
} else {
builder
.orderBy(orderColumn, orderDir)
.orderBy('custom_fields.name', orderDir);
}
}
}
}
);
);
});
}
async function serverValidate(context, listId, data) {
@ -374,7 +382,7 @@ async function removeTx(tx, context, listId, id) {
table.dropColumn(existing.column);
});
await segments.removeRulesByFieldIdTx(tx, context, listId, id);
await segments.removeRulesByColumnTx(tx, context, listId, existing.column);
}
}
@ -391,11 +399,12 @@ async function removeAllByListIdTx(tx, context, listId) {
}
}
module.exports = {
// This is to handle circular dependency with segments.js
Object.assign(module.exports, {
hash,
getById,
list,
listTx,
listByOrderListTx,
listDTAjax,
listGroupedDTAjax,
@ -404,4 +413,4 @@ module.exports = {
remove,
removeAllByListIdTx,
serverValidate
};
});

View file

@ -6,16 +6,215 @@ const interoperableErrors = require('../shared/interoperable-errors');
const shares = require('./shares');
const { enforce, filterObject } = require('../lib/helpers');
const hasher = require('node-object-hash')();
const moment = require('moment');
const fields = require('./fields');
const { parseDate, parseBirthday, DateFormat } = require('../shared/date');
const allowedKeys = new Set(['name', 'settings']);
const predefColumns = [
{
column: 'email',
type: 'text'
},
{
column: 'opt_in_country',
type: 'text'
},
{
column: 'created',
type: 'date'
},
{
column: 'latest_open',
type: 'date'
},
{
column: 'latest_click',
type: 'date'
}
];
const compositeRuleTypes = {
all: {
addQuery: (query, rules, addSubQuery) => {
for (const rule of rules) {
query.where(function() {
addSubQuery(this, rule);
});
}
}
},
some: {
addQuery: (query, rules, addSubQuery) => {
for (const rule of rules) {
query.orWhere(function() {
addSubQuery(this, rule);
});
}
}
},
none: {
addQuery: (query, rules, addSubQuery) => {
for (const rule of rules) {
query.whereNot(function() {
addSubQuery(this, rule);
});
}
}
},
};
const primitiveRuleTypes = {
text: {},
website: {},
number: {},
date: {},
birthday: {},
option: {},
'dropdown-enum': {},
'radio-enum': {}
};
function stringValueSettings(sqlOperator, allowEmpty) {
return {
validate: rule => {
enforce(typeof rule.value === 'string', 'Invalid value type in rule');
enforce(allowEmpty || rule.value, 'Value in rule must not be empty');
},
addQuery: (query, rule) => query.where(rule.column, sqlOperator, rule.value)
};
}
function numberValueSettings(sqlOperator) {
return {
validate: rule => {
enforce(typeof rule.value === 'number', 'Invalid value type in rule');
},
addQuery: (query, rule) => query.where(rule.column, sqlOperator, rule.value)
};
}
function dateValueSettings(thisDaySqlOperator, nextDaySqlOperator) {
return {
validate: rule => {
const date = moment.utc(rule.value);
enforce(date.isValid(), 'Invalid date value');
},
addQuery: (query, rule) => {
const thisDay = moment.utc(rule.value).startOf('day');
const nextDay = moment(thisDay).add(1, 'days');
if (thisDaySqlOperator) {
query.where(rule.column, thisDaySqlOperator, thisDay.toDate())
}
if (nextDaySqlOperator) {
query.where(rule.column, nextDaySqlOperator, nextDay.toDate());
}
}
};
}
function dateRelativeValueSettings(todaySqlOperator, tomorrowSqlOperator) {
return {
validate: rule => {
enforce(typeof rule.value === 'number', 'Invalid value type in rule');
},
addQuery: (query, rule) => {
const todayWithOffset = moment.utc().startOf('day').add(rule.value, 'days');
const tomorrowWithOffset = moment(todayWithOffset).add(1, 'days');
if (todaySqlOperator) {
query.where(rule.column, todaySqlOperator, todayWithOffset.toDate())
}
if (tomorrowSqlOperator) {
query.where(rule.column, tomorrowSqlOperator, tomorrowWithOffset.toDate());
}
}
};
}
function optionValueSettings(value) {
return {
validate: rule => {},
addQuery: (query, rule) => query.where(rule.column, value)
};
}
primitiveRuleTypes.text.eq = stringValueSettings('=', true);
primitiveRuleTypes.text.like = stringValueSettings('LIKE', true);
primitiveRuleTypes.text.re = stringValueSettings('REGEXP', true);
primitiveRuleTypes.text.lt = stringValueSettings('<', false);
primitiveRuleTypes.text.le = stringValueSettings('<=', false);
primitiveRuleTypes.text.gt = stringValueSettings('>', false);
primitiveRuleTypes.text.ge = stringValueSettings('>=', false);
primitiveRuleTypes.website.eq = stringValueSettings('=', true);
primitiveRuleTypes.website.like = stringValueSettings('LIKE', true);
primitiveRuleTypes.website.re = stringValueSettings('REGEXP', true);
primitiveRuleTypes.number.eq = numberValueSettings('=');
primitiveRuleTypes.number.lt = numberValueSettings('<');
primitiveRuleTypes.number.le = numberValueSettings('<=');
primitiveRuleTypes.number.gt = numberValueSettings('>');
primitiveRuleTypes.number.ge = numberValueSettings('>=');
primitiveRuleTypes.date.eq = dateValueSettings('>=', '<');
primitiveRuleTypes.date.lt = dateValueSettings('<', null);
primitiveRuleTypes.date.le = dateValueSettings(null, '<');
primitiveRuleTypes.date.gt = dateValueSettings(null, '>=');
primitiveRuleTypes.date.ge = dateValueSettings('>=', null);
primitiveRuleTypes.date.eqTodayPlusDays = dateRelativeValueSettings('>=', '<');
primitiveRuleTypes.date.ltTodayPlusDays = dateRelativeValueSettings('<', null);
primitiveRuleTypes.date.leTodayPlusDays = dateRelativeValueSettings(null, '<');
primitiveRuleTypes.date.gtTodayPlusDays = dateRelativeValueSettings(null, '>=');
primitiveRuleTypes.date.geTodayPlusDays = dateRelativeValueSettings('>=', null);
primitiveRuleTypes.birthday.eq = dateValueSettings('>=', '<');
primitiveRuleTypes.birthday.lt = dateValueSettings('<', null);
primitiveRuleTypes.birthday.le = dateValueSettings(null, '<');
primitiveRuleTypes.birthday.gt = dateValueSettings(null, '>=');
primitiveRuleTypes.birthday.ge = dateValueSettings('>=', null);
primitiveRuleTypes.option.isTrue = optionValueSettings(true);
primitiveRuleTypes.option.isFalse = optionValueSettings(false);
primitiveRuleTypes['dropdown-enum'].eq = stringValueSettings('=', true);
primitiveRuleTypes['dropdown-enum'].like = stringValueSettings('LIKE', true);
primitiveRuleTypes['dropdown-enum'].re = stringValueSettings('REGEXP', true);
primitiveRuleTypes['dropdown-enum'].lt = stringValueSettings('<', false);
primitiveRuleTypes['dropdown-enum'].le = stringValueSettings('<=', false);
primitiveRuleTypes['dropdown-enum'].gt = stringValueSettings('>', false);
primitiveRuleTypes['dropdown-enum'].ge = stringValueSettings('>=', false);
primitiveRuleTypes['radio-enum'].eq = stringValueSettings('=', true);
primitiveRuleTypes['radio-enum'].like = stringValueSettings('LIKE', true);
primitiveRuleTypes['radio-enum'].re = stringValueSettings('REGEXP', true);
primitiveRuleTypes['radio-enum'].lt = stringValueSettings('<', false);
primitiveRuleTypes['radio-enum'].le = stringValueSettings('<=', false);
primitiveRuleTypes['radio-enum'].gt = stringValueSettings('>', false);
primitiveRuleTypes['radio-enum'].ge = stringValueSettings('>=', false);
function hash(entity) {
return hasher.hash(filterObject(entity, allowedKeys));
}
async function listDTAjax(context, listId, params) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageSegments']);
return await dtHelpers.ajaxListTx(
tx,
@ -30,7 +229,7 @@ async function listDTAjax(context, listId, params) {
async function list(context, listId) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageSegments']);
return await tx('segments').select(['id', 'name']).where('list', listId).orderBy('name', 'asc');
});
@ -38,18 +237,53 @@ async function list(context, listId) {
async function getById(context, listId, id) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageSegments']);
const entity = await tx('segments').where({id, list: listId}).first();
entity.settings = JSON.parse(entity.settings);
return entity;
});
}
async function _validateAndPreprocess(tx, listId, entity, isCreate) {
enforce(entity.name, 'Name must be present');
enforce(entity.settings, 'Settings must be present');
enforce(entity.settings.rootRule, 'Root rule must be present in setting');
enforce(entity.settings.rootRule.type in compositeRuleTypes, 'Root rule must be composite');
const flds = await fields.listTx(tx, listId);
const allowedFlds = [
...predefColumns,
...flds.filter(fld => fld.type in primitiveRuleTypes)
];
const fieldsByColumn = {};
for (const fld of allowedFlds) {
fieldsByColumn[fld.column] = fld;
}
function validateRule(rule) {
if (rule.type in compositeRuleTypes) {
for (const childRule of rule.rules) {
validateRule(childRule);
}
} else {
const colType = fieldsByColumn[rule.column].type;
primitiveRuleTypes[colType][rule.type].validate(rule);
}
}
validateRule(entity.settings.rootRule);
entity.settings = JSON.stringify(entity.settings);
}
async function create(context, listId, entity) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
entity.settings = JSON.stringify(entity.settings);
await _validateAndPreprocess(tx, listId, entity, true);
const filteredEntity = filterObject(entity, allowedKeys);
filteredEntity.list = listId;
@ -77,9 +311,9 @@ async function updateWithConsistencyCheck(context, listId, entity) {
throw new interoperableErrors.ChangedError();
}
entity.settings = JSON.stringify(entity.settings);
await _validateAndPreprocess(tx, listId, entity, false);
await tx('segments').where('id', entity.id).update(filterObject(entity, allowedKeys));
await tx('segments').where({list: listId, id: entity.id}).update(filterObject(entity, allowedKeys));
});
}
@ -88,7 +322,7 @@ async function removeTx(tx, context, listId, id) {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
// The listId "where" is here to prevent deleting segment of a list for which a user does not have permission
await tx('segments').where({list: listId, id: id}).del();
await tx('segments').where({list: listId, id}).del();
}
async function remove(context, listId, id) {
@ -100,15 +334,70 @@ async function remove(context, listId, id) {
async function removeAllByListIdTx(tx, context, listId) {
const entities = await tx('segments').where('list', listId).select(['id']);
for (const entity of entities) {
await removeTx(tx, context, entity.id);
await removeTx(tx, context, listId, entity.id);
}
}
async function removeRulesByFieldIdTx(tx, context, listId, fieldId) {
// FIXME
async function removeRulesByColumnTx(tx, context, listId, column) {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
function pruneChildRules(rule) {
if (rule.type in compositeRuleTypes) {
const newRules = [];
for (const childRule of rule.rules) {
if (childRule.column !== column) {
pruneChildRules(childRule);
newRules.push(childRule);
}
}
rule.rules = newRules;
}
}
const entities = await tx('segments').where({list: listId});
for (const entity of entities) {
const settings = JSON.parse(entity.settings);
pruneChildRules(settings.rootRule);
await tx('segments').where({list: listId, id: entity.id}).update('settings', JSON.stringify(settings));
}
}
module.exports = {
async function getQueryGeneratorTx(tx, listId, id) {
const flds = await fields.listTx(tx, listId);
const allowedFlds = [
...predefColumns,
...flds.filter(fld => fld.type in primitiveRuleTypes)
];
const fieldsByColumn = {};
for (const fld of allowedFlds) {
fieldsByColumn[fld.column] = fld;
}
const entity = await tx('segments').where({id, list: listId}).first();
const settings = JSON.parse(entity.settings);
function processRule(query, rule) {
if (rule.type in compositeRuleTypes) {
compositeRuleTypes[rule.type].addQuery(query, rule.rules, (subQuery, childRule) => {
processRule(subQuery, childRule);
});
} else {
const colType = fieldsByColumn[rule.column].type;
primitiveRuleTypes[colType][rule.type].addQuery(query, rule);
}
}
return query => processRule(query, settings.rootRule);
}
// This is to handle circular dependency with fields.js
Object.assign(module.exports, {
hash,
listDTAjax,
list,
@ -117,5 +406,6 @@ module.exports = {
updateWithConsistencyCheck,
remove,
removeAllByListIdTx,
removeRulesByFieldIdTx
};
removeRulesByColumnTx,
getQueryGeneratorTx
});

View file

@ -6,6 +6,8 @@ const interoperableErrors = require('../shared/interoperable-errors');
const shares = require('./shares');
const fields = require('./fields');
const { SubscriptionStatus } = require('../shared/lists');
const segments = require('./segments');
const allowedKeysBase = new Set(['cid', 'email']);
@ -18,16 +20,23 @@ function hash(entity) {
}
async function listDTAjax(context, listId, params) {
async function listDTAjax(context, listId, segmentId, params) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
const flds = await fields.listByOrderListTx(tx, listId, ['column']);
const addSegmentQuery = segmentId ? await segments.getQueryGeneratorTx(tx, listId, segmentId) : () => {};
return await dtHelpers.ajaxListTx(
tx,
params,
builder => builder.from(`subscription__${listId}`),
builder => {
const query = builder.from(`subscription__${listId}`);
query.where(function() {
addSegmentQuery(this);
});
return query;
},
['id', 'cid', 'email', 'status', 'created', ...flds.map(fld => fld.column)]
);
});