New project structure

Beta of extract.js for extracting english locale

server/models/blacklist.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const dtHelpers = require('../lib/dt-helpers');
+const shares = require('./shares');
+const tools = require('../lib/tools');
+
+async function listDTAjax(context, params) {
+    shares.enforceGlobalPermission(context, 'manageBlacklist');
+
+    return await dtHelpers.ajaxList(
+        params,
+        builder => builder
+            .from('blacklist'),
+        ['blacklist.email']
+    );
+}
+
+async function search(context, offset, limit, search) {
+    return await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'manageBlacklist');
+
+        search = '%' + search + '%';
+
+        const count = await tx('blacklist').where('email', 'like', search).count('* as count').first().count;
+
+        const rows = await tx('blacklist').where('email', 'like', search).offset(offset).limit(limit);
+
+        return {
+            emails: rows.map(row => row.email),
+            total: count
+        };
+    });
+}
+
+async function add(context, email) {
+    return await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'manageBlacklist');
+
+        const existing = await tx('blacklist').where('email', email).first();
+        if (!existing) {
+            await tx('blacklist').insert({email});
+        }
+    });
+}
+
+async function remove(context, email) {
+    shares.enforceGlobalPermission(context, 'manageBlacklist');
+    await knex('blacklist').where('email', email).del();
+}
+
+async function isBlacklisted(email) {
+    const existing = await knex('blacklist').where('email', email).first();
+    return !!existing;
+}
+
+async function serverValidate(context, data) {
+    shares.enforceGlobalPermission(context, 'manageBlacklist');
+    const result = {};
+
+    if (data.email) {
+        const user = await knex('blacklist').where('email', data.email).first();
+
+        result.email = {};
+        result.email.invalid = await tools.validateEmail(data.email) !== 0;
+        result.email.exists = !!user;
+    }
+
+    return result;
+}
+
+module.exports.listDTAjax = listDTAjax;
+module.exports.add = add;
+module.exports.remove = remove;
+module.exports.search = search;
+module.exports.isBlacklisted = isBlacklisted;
+module.exports.serverValidate = serverValidate;

server/models/campaigns.js

@@ -0,0 +1,832 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const files = require('./files');
+const templates = require('./templates');
+const { CampaignStatus, CampaignSource, CampaignType, getSendConfigurationPermissionRequiredForSend} = require('../../shared/campaigns');
+const sendConfigurations = require('./send-configurations');
+const triggers = require('./triggers');
+const {SubscriptionStatus} = require('../../shared/lists');
+const subscriptions = require('./subscriptions');
+const segments = require('./segments');
+const senders = require('../lib/senders');
+
+const allowedKeysCommon = ['name', 'description', 'segment', 'namespace',
+    'send_configuration', 'from_name_override', 'from_email_override', 'reply_to_override', 'subject_override', 'data', 'click_tracking_disabled', 'open_tracking_disabled', 'unsubscribe_url'];
+
+const allowedKeysCreate = new Set(['type', 'source', ...allowedKeysCommon]);
+const allowedKeysCreateRssEntry = new Set(['type', 'source', 'parent', ...allowedKeysCommon]);
+const allowedKeysUpdate = new Set([...allowedKeysCommon]);
+
+const Content = {
+    ALL: 0,
+    WITHOUT_SOURCE_CUSTOM: 1,
+    ONLY_SOURCE_CUSTOM: 2,
+    RSS_ENTRY: 3,
+    SETTINGS_WITH_STATS: 4
+};
+
+function hash(entity, content) {
+    let filteredEntity;
+
+    if (content === Content.ALL) {
+        filteredEntity = filterObject(entity, allowedKeysUpdate);
+        filteredEntity.lists = entity.lists;
+
+    } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+        filteredEntity = filterObject(entity, allowedKeysUpdate);
+        filteredEntity.lists = entity.lists;
+        filteredEntity.data = {...filteredEntity.data};
+        delete filteredEntity.data.sourceCustom;
+
+    } else if (content === Content.ONLY_SOURCE_CUSTOM) {
+        filteredEntity = {
+            data: {
+                sourceCustom: entity.data.sourceCustom
+            }
+        };
+    }
+
+    return hasher.hash(filteredEntity);
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .whereNull('campaigns.parent'),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.status', 'campaigns.scheduled', 'campaigns.source', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+async function listChildrenDTAjax(context, campaignId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .where('campaigns.parent', campaignId),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.status', 'campaigns.scheduled', 'campaigns.source', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+
+async function listWithContentDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .whereIn('campaigns.source', [CampaignSource.CUSTOM, CampaignSource.CUSTOM_FROM_TEMPLATE, CampaignSource.CUSTOM_FROM_CAMPAIGN]),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+async function listOthersWhoseListsAreIncludedDTAjax(context, campaignId, listIds, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('campaigns')
+            .innerJoin('namespaces', 'namespaces.id', 'campaigns.namespace')
+            .whereNot('campaigns.id', campaignId)
+            .whereNotExists(qry => qry.from('campaign_lists').whereRaw('campaign_lists.campaign = campaigns.id').whereNotIn('campaign_lists.list', listIds)),
+        ['campaigns.id', 'campaigns.name', 'campaigns.cid', 'campaigns.description', 'campaigns.type', 'campaigns.created', 'namespaces.name']
+    );
+}
+
+async function listTestUsersDTAjax(context, campaignId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'view');
+
+        /*
+        This is supposed to produce queries like this:
+
+        select * from (
+          (select `subscription__1`.`email`, `subscription__1`.`cid`, 1 AS list, NULL AS segment from `subscription__1` where `subscription__1`.`status` = 1 and `subscription__1`.`is_test` = true)
+        UNION ALL
+          (select `subscription__2`.`email`, `subscription__2`.`cid`, 2 AS list, NULL AS segment from `subscription__2` where `subscription__2`.`status` = 1 and `subscription__2`.`is_test` = true)
+        ) as `test_subscriptions` inner join `lists` on `test_subscriptions`.`list` = `lists`.`id` inner join `segments` on `test_subscriptions`.`segment` = `segments`.`id`
+          inner join `namespaces` on `lists`.`namespace` = `namespaces`.`id`
+
+        This was too much for Knex, so we partially construct these queries directly as strings;
+        */
+
+        const subsQrys = [];
+        const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+
+        for (const cpgList of cpgLists) {
+            const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
+            const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
+
+            const sqlQry = knex.from(subsTable)
+                .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+                .where(subsTable + '.is_test', true)
+                .where(function() {
+                    addSegmentQuery(this);
+                })
+                .select([subsTable + '.email', subsTable + '.cid', knex.raw('? AS list', [cpgList.list]), knex.raw('? AS segment', [cpgList.segment])])
+                .toSQL().toNative();
+
+            subsQrys.push(sqlQry);
+        }
+
+        if (subsQrys.length > 0) {
+            let subsQry;
+
+            if (subsQrys.length === 1) {
+                const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`'
+                subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
+
+            } else {
+                const subsUnionSql = '(' +
+                    subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
+                    ') as `test_subscriptions`';
+                const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+                subsQry = knex.raw(subsUnionSql, subsUnionBindings);
+            }
+
+            return await dtHelpers.ajaxListWithPermissions(
+                context,
+                [{ entityTypeId: 'list', requiredOperations: ['viewSubscriptions'], column: 'subs.list_id' }],
+                params,
+                builder => {
+                    return builder.from(function () {
+                        return this.from(subsQry)
+                            .innerJoin('lists', 'test_subscriptions.list', 'lists.id')
+                            .innerJoin('namespaces', 'lists.namespace', 'namespaces.id')
+                            .select([
+                                knex.raw('CONCAT_WS(":", lists.cid, test_subscriptions.cid) AS cid'),
+                                'test_subscriptions.email', 'test_subscriptions.cid AS subscription_cid', 'lists.cid AS list_cid',
+                                'lists.name as list_name', 'namespaces.name AS namespace_name', 'lists.id AS list_id'
+                            ])
+                            .as('subs');
+                    });
+                },
+                [ 'subs.cid', 'subs.email', 'subs.subscription_cid', 'subs.list_cid', 'subs.list_name', 'subs.namespace_name' ]
+            );
+
+        } else {
+            const result = {
+                draw: params.draw,
+                recordsTotal: 0,
+                recordsFiltered: 0,
+                data: []
+            };
+
+            return result;
+        }
+    });
+}
+
+async function getTrackingSettingsByCidTx(tx, cid) {
+    const entity = await tx('campaigns').where('campaigns.cid', cid)
+        .select([
+            'campaigns.id', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled'
+        ])
+        .first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    return entity;
+}
+
+async function rawGetByTx(tx, key, id) {
+    const entity = await tx('campaigns').where('campaigns.' + key, id)
+        .leftJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
+        .groupBy('campaigns.id')
+        .select([
+            'campaigns.id', 'campaigns.cid', 'campaigns.name', 'campaigns.description', 'campaigns.namespace', 'campaigns.status', 'campaigns.type', 'campaigns.source',
+            'campaigns.send_configuration', 'campaigns.from_name_override', 'campaigns.from_email_override', 'campaigns.reply_to_override', 'campaigns.subject_override',
+            'campaigns.data', 'campaigns.click_tracking_disabled', 'campaigns.open_tracking_disabled', 'campaigns.unsubscribe_url', 'campaigns.scheduled',
+            knex.raw(`GROUP_CONCAT(CONCAT_WS(\':\', campaign_lists.list, campaign_lists.segment) ORDER BY campaign_lists.id SEPARATOR \';\') as lists`)
+        ])
+        .first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    if (entity.lists) {
+        entity.lists = entity.lists.split(';').map(x => {
+            const entries = x.split(':');
+            const list = Number.parseInt(entries[0]);
+            const segment = entries[1] ? Number.parseInt(entries[1]) : null;
+            return {list, segment};
+        });
+    } else {
+        entity.lists = [];
+    }
+
+    entity.data = JSON.parse(entity.data);
+
+    return entity;
+}
+
+async function getByIdTx(tx, context, id, withPermissions = true, content = Content.ALL) {
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'view');
+
+    let entity = await rawGetByTx(tx, 'id', id);
+
+    if (content === Content.ALL || content === Content.RSS_ENTRY) {
+        // Return everything
+
+    } else if (content === Content.SETTINGS_WITH_STATS) {
+        delete entity.data.sourceCustom;
+
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'viewStats');
+
+        const unsentQryGen = await getSubscribersQueryGeneratorTx(tx, id, true);
+        if (unsentQryGen) {
+            const res = await unsentQryGen(tx).count('* AS subscriptionsToSend').first();
+            entity.subscriptionsToSend = res.subscriptionsToSend;
+        }
+
+        const totalQryGen = await getSubscribersQueryGeneratorTx(tx, id, false);
+        if (totalQryGen) {
+            const res = await totalQryGen(tx).count('* AS subscriptionsTotal').first();
+            entity.subscriptionsTotal = res.subscriptionsTotal;
+        }
+
+    } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+        delete entity.data.sourceCustom;
+
+    } else if (content === Content.ONLY_SOURCE_CUSTOM) {
+        entity = {
+            id: entity.id,
+            send_configuration: entity.send_configuration,
+
+            data: {
+                sourceCustom: entity.data.sourceCustom
+            }
+        };
+    }
+
+    if (withPermissions) {
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'campaign', id);
+    }
+
+    return entity;
+}
+
+async function getById(context, id, withPermissions = true, content = Content.ALL) {
+    return await knex.transaction(async tx => {
+        return await getByIdTx(tx, context, id, withPermissions, content);
+    });
+}
+
+async function _validateAndPreprocess(tx, context, entity, isCreate, content) {
+    if (content === Content.ALL || content === Content.WITHOUT_SOURCE_CUSTOM || content === Content.RSS_ENTRY) {
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        if (isCreate) {
+            enforce(entity.type === CampaignType.REGULAR || entity.type === CampaignType.RSS || entity.type === CampaignType.TRIGGERED ||
+                    (content === Content.RSS_ENTRY && entity.type === CampaignType.RSS_ENTRY),
+                'Unknown campaign type');
+
+            if (entity.source === CampaignSource.TEMPLATE || entity.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
+                await shares.enforceEntityPermissionTx(tx, context, 'template', entity.data.sourceTemplate, 'view');
+            }
+
+            enforce(Number.isInteger(entity.source));
+            enforce(entity.source >= CampaignSource.MIN && entity.source <= CampaignSource.MAX, 'Unknown campaign source');
+        }
+
+        for (const lstSeg of entity.lists) {
+            await shares.enforceEntityPermissionTx(tx, context, 'list', lstSeg.list, 'view');
+
+            if (lstSeg.segment) {
+                // Check that the segment under the list exists
+                await segments.getByIdTx(tx, context, lstSeg.list, lstSeg.segment);
+            }
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', entity.send_configuration, 'viewPublic');
+    }
+}
+
+function convertFileURLs(sourceCustom, fromEntityType, fromEntityId, toEntityType, toEntityId) {
+
+    function convertText(text) {
+        if (text) {
+            const fromUrl = `/files/${fromEntityType}/file/${fromEntityId}`;
+            const toUrl = `/files/${toEntityType}/file/${toEntityId}`;
+
+            const encodedFromUrl = encodeURIComponent(fromUrl);
+            const encodedToUrl = encodeURIComponent(toUrl);
+
+            text = text.split('[URL_BASE]' + fromUrl).join('[URL_BASE]' + toUrl);
+            text = text.split('[SANDBOX_URL_BASE]' + fromUrl).join('[SANDBOX_URL_BASE]' + toUrl);
+            text = text.split('[ENCODED_URL_BASE]' + encodedFromUrl).join('[ENCODED_URL_BASE]' + encodedToUrl);
+            text = text.split('[ENCODED_SANDBOX_URL_BASE]' + encodedFromUrl).join('[ENCODED_SANDBOX_URL_BASE]' + encodedToUrl);
+        }
+
+        return text;
+    }
+
+    sourceCustom.html = convertText(sourceCustom.html);
+    sourceCustom.text = convertText(sourceCustom.text);
+
+    if (sourceCustom.type === 'mosaico' || sourceCustom.type === 'mosaicoWithFsTemplate') {
+        sourceCustom.data.model = convertText(sourceCustom.data.model);
+        sourceCustom.data.model = convertText(sourceCustom.data.model);
+        sourceCustom.data.metadata = convertText(sourceCustom.data.metadata);
+    }
+}
+
+async function _createTx(tx, context, entity, content) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createCampaign');
+
+        let copyFilesFrom = null;
+        if (entity.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
+            copyFilesFrom = {
+                entityType: 'template',
+                entityId: entity.data.sourceTemplate
+            };
+
+            const template = await templates.getByIdTx(tx, context, entity.data.sourceTemplate, false);
+
+            entity.data.sourceCustom = {
+                type: template.type,
+                data: template.data,
+                html: template.html,
+                text: template.text
+            };
+
+        } else if (entity.source === CampaignSource.CUSTOM_FROM_CAMPAIGN) {
+            copyFilesFrom = {
+                entityType: 'campaign',
+                entityId: entity.data.sourceCampaign
+            };
+
+            const sourceCampaign = await getByIdTx(tx, context, entity.data.sourceCampaign, false);
+            enforce(sourceCampaign.source === CampaignSource.CUSTOM || sourceCampaign.source === CampaignSource.CUSTOM_FROM_TEMPLATE || sourceCampaign.source === CampaignSource.CUSTOM_FROM_CAMPAIGN, 'Incorrect source type of the source campaign.');
+
+            entity.data.sourceCustom = sourceCampaign.data.sourceCustom;
+        }
+
+        await _validateAndPreprocess(tx, context, entity, true, content);
+
+        const filteredEntity = filterObject(entity, entity.type === CampaignType.RSS_ENTRY ? allowedKeysCreateRssEntry : allowedKeysCreate);
+        filteredEntity.cid = shortid.generate();
+
+        const data = filteredEntity.data;
+
+        filteredEntity.data = JSON.stringify(filteredEntity.data);
+
+        if (filteredEntity.type === CampaignType.RSS || filteredEntity.type === CampaignType.TRIGGERED) {
+            filteredEntity.status = CampaignStatus.ACTIVE;
+        } else if (filteredEntity.type === CampaignType.RSS_ENTRY) {
+            filteredEntity.status = CampaignStatus.SCHEDULED;
+        } else {
+            filteredEntity.status = CampaignStatus.IDLE;
+        }
+
+        const ids = await tx('campaigns').insert(filteredEntity);
+        const id = ids[0];
+
+        await tx('campaign_lists').insert(entity.lists.map(x => ({campaign: id, ...x})));
+
+        if (entity.source === CampaignSource.TEMPLATE) {
+            await tx('template_dep_campaigns').insert({
+               campaign: id,
+               template: entity.data.sourceTemplate
+            });
+        }
+
+        if (filteredEntity.parent) {
+            await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: id, parentId: filteredEntity.parent });
+        } else {
+            await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: id });
+        }
+
+        if (copyFilesFrom) {
+            await files.copyAllTx(tx, context, copyFilesFrom.entityType, 'file', copyFilesFrom.entityId, 'campaign', 'file', id);
+
+            convertFileURLs(data.sourceCustom, copyFilesFrom.entityType, copyFilesFrom.entityId, 'campaign', id);
+            await tx('campaigns')
+                .update({
+                    data: JSON.stringify(data)
+                }).where('id', id);
+        }
+
+        return id;
+    });
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        return await _createTx(tx, context, entity, Content.ALL);
+    });
+}
+
+async function createRssTx(tx, context, entity) {
+    return await _createTx(tx, context, entity, Content.RSS_ENTRY);
+}
+
+async function updateWithConsistencyCheck(context, entity, content) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.id, 'edit');
+
+        const existing = await rawGetByTx(tx, 'id', entity.id);
+
+        const existingHash = hash(existing, content);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, context, entity, false, content);
+
+        let filteredEntity = filterObject(entity, allowedKeysUpdate);
+        if (content === Content.ALL) {
+            await namespaceHelpers.validateMove(context, entity, existing, 'campaign', 'createCampaign', 'delete');
+
+        } else if (content === Content.WITHOUT_SOURCE_CUSTOM) {
+            filteredEntity.data.sourceCustom = existing.data.sourceCustom;
+            await namespaceHelpers.validateMove(context, filteredEntity, existing, 'campaign', 'createCampaign', 'delete');
+
+        } else if (content === Content.ONLY_SOURCE_CUSTOM) {
+            const data = existing.data;
+            data.sourceCustom = filteredEntity.data.sourceCustom;
+            filteredEntity = {
+                data
+            };
+        }
+
+        if (content === Content.ALL || content === Content.WITHOUT_SOURCE_CUSTOM) {
+            await tx('campaign_lists').where('campaign', entity.id).del();
+            await tx('campaign_lists').insert(entity.lists.map(x => ({campaign: entity.id, ...x})));
+
+            if (existing.source === CampaignSource.TEMPLATE) {
+                await tx('template_dep_campaigns')
+                    .where('campaign', entity.id)
+                    .update('template', entity.data.sourceTemplate);
+            }
+        }
+
+        filteredEntity.data = JSON.stringify(filteredEntity.data);
+        await tx('campaigns').where('id', entity.id).update(filteredEntity);
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: entity.id });
+    });
+}
+
+async function _removeTx(tx, context, id, existing = null) {
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'delete');
+
+    if (!existing) {
+        existing = await tx('campaigns').where('id', id).select(['id', 'status', 'type']).first();
+    }
+
+    if (existing.status === CampaignStatus.SENDING) {
+        return new interoperableErrors.InvalidStateError;
+    }
+
+    enforce(existing.type === CampaignType.REGULAR || existing.type === CampaignType.RSS || existing.type === CampaignType.TRIGGERED, 'This campaign cannot be removed by user.');
+
+    const childCampaigns = await tx('campaigns').where('parent', id).select(['id', 'status', 'type']);
+    for (const childCampaign of childCampaigns) {
+        await _removeTx(tx, contect, childCampaign.id, childCampaign);
+    }
+
+    await files.removeAllTx(tx, context, 'campaign', 'file', id);
+    await files.removeAllTx(tx, context, 'campaign', 'attachment', id);
+
+    await tx('campaign_lists').where('campaign', id).del();
+    await tx('campaign_messages').where('campaign', id).del();
+    await tx('campaign_links').where('campaign', id).del();
+
+    await triggers.removeAllByCampaignIdTx(tx, context, id);
+
+    await tx('template_dep_campaigns')
+        .where('campaign', id)
+        .del();
+
+    await tx('campaigns').where('id', id).del();
+}
+
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await _removeTx(tx, context, id);
+    });
+}
+
+async function enforceSendPermissionTx(tx, context, campaignId) {
+    const campaign = await getByIdTx(tx, context, campaignId, false);
+    const sendConfiguration = await sendConfigurations.getByIdTx(tx, context, campaign.send_configuration, false, false);
+
+    const requiredPermission = getSendConfigurationPermissionRequiredForSend(campaign, sendConfiguration);
+
+    await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', campaign.send_configuration, requiredPermission);
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+}
+
+
+// Message API
+
+function getMessageCid(campaignCid, listCid, subscriptionCid) {
+    return [campaignCid, listCid, subscriptionCid].join('.')
+}
+
+async function getMessageByCid(messageCid) {
+    const messageCidElems = messageCid.split('.');
+
+    if (messageCidElems.length !== 3) {
+        return null;
+    }
+
+    const [campaignCid, listCid, subscriptionCid] = messageCidElems;
+
+    await knex.transaction(async tx => {
+        const list = await tx('lists').where('cid', listCid).select('id');
+        const subscrTblName = subscriptions.getSubscriptionTableName(list.id);
+
+        const message = await tx('campaign_messages')
+            .innerJoin('campaigns', 'campaign_messages.campaign', 'campaigns.id')
+            .innerJoin(subscrTblName, subscrTblName + '.id', 'campaign_messages.subscription')
+            .leftJoin('segments', 'segment.id', 'campaign_messages.segment') // This is just to make sure that the respective segment still exists or return null if it doesn't
+            .leftJoin('send_configurations', 'send_configurations.id', 'campaign_messages.send_configuration') // This is just to make sure that the respective send_configuration still exists or return null if it doesn't
+            .where(subscrTblName + '.cid', subscriptionCid)
+            .where('campaigns.cid', campaignCid)
+            .select([
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'segments.id AS segment', 'campaign_messages.subscription',
+                'send_configurations.id AS send_configuration', 'campaign_messages.status', 'campaign_messages.response', 'campaign_messages.response_id',
+                'campaign_messages.updated', 'campaign_messages.created', 'send_configurations.verp_hostname AS verp_hostname'
+            ]);
+
+        if (message) {
+            await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+        }
+
+        return message;
+    });
+}
+
+async function getMessageByResponseId(responseId) {
+    await knex.transaction(async tx => {
+        const message = await tx('campaign_messages')
+            .leftJoin('segments', 'segment.id', 'campaign_messages.segment') // This is just to make sure that the respective segment still exists or return null if it doesn't
+            .leftJoin('send_configurations', 'send_configurations.id', 'campaign_messages.send_configuration') // This is just to make sure that the respective send_configuration still exists or return null if it doesn't
+            .where('campaign_messages.response_id', responseId)
+            .select([
+                'campaign_messages.id', 'campaign_messages.campaign', 'campaign_messages.list', 'segments.id AS segment', 'campaign_messages.subscription',
+                'send_configurations.id AS send_configuration', 'campaign_messages.status', 'campaign_messages.response', 'campaign_messages.response_id',
+                'campaign_messages.updated', 'campaign_messages.created', 'send_configurations.verp_hostname AS verp_hostname'
+            ]);
+
+        if (message) {
+            await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+        }
+
+        return message;
+    });
+}
+
+const statusFieldMapping = {
+    [SubscriptionStatus.UNSUBSCRIBED]: 'unsubscribed',
+    [SubscriptionStatus.BOUNCED]: 'bounced',
+    [SubscriptionStatus.COMPLAINED]: 'complained'
+};
+
+async function _changeStatusByMessageTx(tx, context, message, subscriptionStatus) {
+    enforce(subscriptionStatus !== SubscriptionStatus.SUBSCRIBED);
+
+    if (message.status === SubscriptionStatus.SUBSCRIBED) {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+
+        if (!subscriptionStatus in statusFieldMapping) {
+            throw new Error('Unrecognized message status');
+        }
+
+        const statusField = statusFieldMapping[subscriptionStatus];
+
+        if (message.status === SubscriptionStatus.SUBSCRIBED) {
+            await tx('campaigns').increment(statusField, 1).where('id', message.campaign);
+        }
+
+        await tx('campaign_messages')
+            .where('id', message.id)
+            .update({
+                status: subscriptionStatus,
+                updated: knex.fn.now()
+            });
+    }
+}
+
+async function changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaignCid, listId, subscriptionId, subscriptionStatus) {
+    const campaign = await tx('campaigns').where('cid', campaignCid);
+    const message = await tx('campaign_messages')
+        .innerJoin('campaigns', 'campaign_messages.campaign', 'campaigns.id')
+        .where('campaigns.cid', campaignCid)
+        .where({subscription: subscriptionId, list: listId});
+
+    if (!message) {
+        throw new Error('Invalid campaign.')
+    }
+
+    await _changeStatusByMessageTx(tx, context, message, subscriptionStatus);
+}
+
+async function changeStatusByMessage(context, message, subscriptionStatus, updateSubscription) {
+    await knex.transaction(async tx => {
+        if (updateSubscription) {
+            await subscriptions.changeStatusTx(tx, context, message.list, message.subscription, subscriptionStatus);
+        }
+
+        await _changeStatusByMessageTx(tx, context, message, subscriptionStatus);
+    });
+}
+
+async function updateMessageResponse(context, message, response, responseId) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', message.campaign, 'manageMessages');
+
+        await tx('campaign_messages').where('id', message.id).update({
+            response,
+            response_id: responseId
+        });
+    });
+}
+
+async function getSubscribersQueryGeneratorTx(tx, campaignId, onlyUnsent) {
+    /*
+    This is supposed to produce queries like this:
+
+    select ... from `campaign_lists` inner join (
+        select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (
+            (select `subscription__2`.`email`, 8 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__2` left join
+                (select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 2)
+                    as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__2`.`id` where `subscription__2`.`status` = 1)
+            UNION ALL
+            (select `subscription__1`.`email`, 9 AS campaign_list_id, related_campaign_messages.id IS NOT NULL AS sent from `subscription__1` left join
+                (select * from `campaign_messages` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 1)
+                    as `related_campaign_messages` on `related_campaign_messages`.`subscription` = `subscription__1`.`id` where `subscription__1`.`status` = 1)
+        ) as `pending_subscriptions_all` where `sent` = false group by `email`)
+            as `pending_subscriptions` on `campaign_lists`.`id` = `pending_subscriptions`.`campaign_list_id` where `campaign_lists`.`campaign` = '1'
+
+     This was too much for Knex, so we partially construct these queries directly as strings;
+     */
+
+    const subsQrys = [];
+    const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
+
+    for (const cpgList of cpgLists) {
+        const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
+        const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
+
+        const sqlQry = knex.from(subsTable)
+            .leftJoin(
+                function () {
+                    return this.from('campaign_messages')
+                        .where('campaign_messages.campaign', campaignId)
+                        .where('campaign_messages.list', cpgList.list)
+                        .as('related_campaign_messages');
+                },
+                'related_campaign_messages.subscription', subsTable + '.id')
+            .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+            .where(function() {
+                addSegmentQuery(this);
+            })
+            .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('related_campaign_messages.id IS NOT NULL AS sent')])
+            .toSQL().toNative();
+
+        subsQrys.push(sqlQry);
+    }
+
+    if (subsQrys.length > 0) {
+        let subsQry;
+        const unsentWhere = onlyUnsent ? ' where `sent` = false' : '';
+
+        if (subsQrys.length === 1) {
+            const subsUnionSql = '(select `email`, `campaign_list_id`, `sent` from (' + subsQrys[0].sql + ') as `pending_subscriptions_all`' +  unsentWhere + ') as `pending_subscriptions`'
+            subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
+
+        } else {
+            const subsUnionSql = '(select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (' +
+                subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
+                ') as `pending_subscriptions_all`' +  unsentWhere + ' group by `email`) as `pending_subscriptions`';
+            const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+            subsQry = knex.raw(subsUnionSql, subsUnionBindings);
+        }
+
+        return knx => knx.from('campaign_lists')
+            .where('campaign_lists.campaign', campaignId)
+            .innerJoin(subsQry, 'campaign_lists.id', 'pending_subscriptions.campaign_list_id');
+
+    } else {
+        return null;
+    }
+}
+
+async function _changeStatus(context, campaignId, permittedCurrentStates, newState, invalidStateMessage, scheduled = null) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+
+        const entity = await tx('campaigns').where('id', campaignId).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!permittedCurrentStates.includes(entity.status)) {
+            throw new interoperableErrors.InvalidStateError(invalidStateMessage);
+        }
+
+        await tx('campaigns').where('id', campaignId).update({
+            status: newState,
+            scheduled
+        });
+    });
+
+    senders.scheduleCheck();
+}
+
+
+async function start(context, campaignId, startAt) {
+    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.SCHEDULED, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt);
+}
+
+async function stop(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.SCHEDULED], CampaignStatus.PAUSED, 'Cannot stop campaign until it is in SCHEDULED state');
+}
+
+async function reset(context, campaignId) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'send');
+
+        const entity = await tx('campaigns').where('id', campaignId).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (entity.status !== CampaignStatus.FINISHED && entity.status !== CampaignStatus.PAUSED) {
+            throw new interoperableErrors.InvalidStateError('Cannot reset campaign until it is FINISHED or PAUSED state');
+        }
+
+        await tx('campaigns').where('id', campaignId).update({
+            status: CampaignStatus.IDLE
+        });
+
+        await tx('campaign_messages').where('campaign', campaignId).del();
+        await tx('campaign_links').where('campaign', campaignId).del();
+    });
+}
+
+async function enable(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.INACTIVE], CampaignStatus.ACTIVE, 'Cannot enable campaign unless it is in INACTIVE state');
+}
+
+async function disable(context, campaignId) {
+    await _changeStatus(context, campaignId, [CampaignStatus.ACTIVE], CampaignStatus.INACTIVE, 'Cannot disable campaign unless it is in ACTIVE state');
+}
+
+
+
+module.exports.Content = Content;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listChildrenDTAjax = listChildrenDTAjax;
+module.exports.listWithContentDTAjax = listWithContentDTAjax;
+module.exports.listOthersWhoseListsAreIncludedDTAjax = listOthersWhoseListsAreIncludedDTAjax;
+module.exports.listTestUsersDTAjax = listTestUsersDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.createRssTx = createRssTx;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.enforceSendPermissionTx = enforceSendPermissionTx;
+
+module.exports.getMessageCid = getMessageCid;
+module.exports.getMessageByCid = getMessageByCid;
+module.exports.getMessageByResponseId = getMessageByResponseId;
+
+module.exports.changeStatusByCampaignCidAndSubscriptionIdTx = changeStatusByCampaignCidAndSubscriptionIdTx;
+module.exports.changeStatusByMessage = changeStatusByMessage;
+module.exports.updateMessageResponse = updateMessageResponse;
+
+module.exports.getSubscribersQueryGeneratorTx = getSubscribersQueryGeneratorTx;
+
+module.exports.start = start;
+module.exports.stop = stop;
+module.exports.reset = reset;
+module.exports.enable = enable;
+module.exports.disable = disable;
+
+module.exports.rawGetByTx = rawGetByTx;
+module.exports.getTrackingSettingsByCidTx = getTrackingSettingsByCidTx;

server/models/confirmations.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const shortid = require('shortid');
+
+async function addConfirmation(listId, action, ip, data) {
+    const cid = shortid.generate();
+    await knex('confirmations').insert({
+        cid,
+        list: listId,
+        action,
+        ip,
+        data: JSON.stringify(data || {})
+    });
+
+    return cid;
+}
+
+/*
+ Atomically retrieves confirmation from the database, removes it from the database and returns it.
+ */
+async function takeConfirmation(cid) {
+    return await knex.transaction(async tx => {
+        const entry = await tx('confirmations').select(['cid', 'list', 'action', 'ip', 'data']).where('cid', cid).first();
+
+        if (!entry) {
+            return false;
+        }
+
+        await tx('confirmations').where('cid', cid).del();
+
+        let data;
+        try {
+            data = JSON.parse(entry.data);
+        } catch (err) {
+            data = {};
+        }
+
+        return {
+            list: entry.list,
+            action: entry.action,
+            ip: entry.ip,
+            data
+        };
+    });
+}
+
+module.exports.addConfirmation = addConfirmation;
+module.exports.takeConfirmation = takeConfirmation;

server/models/fields.js

@@ -0,0 +1,828 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const slugify = require('slugify');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const validators = require('../../shared/validators');
+const shortid = require('shortid');
+const segments = require('./segments');
+const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../../shared/date');
+const { getFieldColumn } = require('../../shared/lists');
+const { cleanupFromPost } = require('../lib/helpers');
+const Handlebars = require('handlebars');
+const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('../lib/urls');
+const { getMergeTagsForBases } = require('../../shared/templates');
+
+
+const allowedKeysCreate = new Set(['name', 'key', 'default_value', 'type', 'group', 'settings']);
+const allowedKeysUpdate = new Set(['name', 'key', 'default_value', 'group', 'settings']);
+const hashKeys = allowedKeysCreate;
+
+const fieldTypes = {};
+
+const Cardinality = {
+    SINGLE: 0,
+    MULTIPLE: 1
+};
+
+function render(template, options) {
+    const renderer = Handlebars.compile(template || '');
+    return renderer(options);
+}
+
+fieldTypes.text = {
+    validate: field => {},
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeText',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.website = {
+    validate: field => {},
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeWebsite',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.longtext = {
+    validate: field => {},
+    addColumn: (table, name) => table.text(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeLongtext',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.gpg = {
+    validate: field => {},
+    addColumn: (table, name) => table.text(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeGpg',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => value
+};
+
+fieldTypes.json = {
+    validate: field => {},
+    addColumn: (table, name) => table.json(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeJson',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value,
+    render: (field, value) => {
+        try {
+            if (value === null || value.trim() === '') {
+                return '';
+            }
+
+            let parsed = JSON.parse(value);
+            if (Array.isArray(parsed)) {
+                parsed = {
+                    values: parsed
+                };
+            }
+            return render(field.settings.renderTemplate, parsed);
+        } catch (err) {
+            return err.message;
+        }
+    }
+};
+
+fieldTypes.number = {
+    validate: field => {},
+    addColumn: (table, name) => table.integer(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeNumber',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => Number(value),
+    render: (field, value) => value
+};
+
+fieldTypes['checkbox-grouped'] = {
+    validate: field => {},
+    indexed: true,
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.MULTIPLE,
+    getHbsType: field => 'typeCheckboxGrouped',
+    render: (field, value) => {
+        const subItems = (value || []).map(col => field.groupedOptions[col].name);
+
+        if (field.settings.groupTemplate) {
+            return render(field.settings.groupTemplate, {
+                values: subItems
+            });
+        } else {
+            return subItems.join(', ');
+        }
+    }
+};
+
+fieldTypes['radio-grouped'] = {
+    validate: field => {},
+    indexed: true,
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeRadioGrouped',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes['dropdown-grouped'] = {
+    validate: field => {},
+    indexed: true,
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDropdownGrouped',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes['radio-enum'] = {
+    validate: field => {
+        enforce(field.settings.options, 'Options missing in settings');
+        enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
+    },
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: true,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeRadioEnum',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes['dropdown-enum'] = {
+    validate: field => {
+        enforce(field.settings.options, 'Options missing in settings');
+        enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
+    },
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: true,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDropdownEnum',
+    render: (field, value) => {
+        const fld = field.groupedOptions[value];
+        return fld ? fld.name : '';
+    }
+};
+
+fieldTypes.option = {
+    validate: field => {},
+    addColumn: (table, name) => table.boolean(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    parsePostValue: (field, value) => !(['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0)
+};
+
+fieldTypes['date'] = {
+    validate: field => {
+        enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
+    },
+    addColumn: (table, name) => table.dateTime(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDate' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
+    forHbs: (field, value) => formatDate(field.settings.dateFormat, value),
+    parsePostValue: (field, value) => parseDate(field.settings.dateFormat, value),
+    render: (field, value) => value !== null ? formatDate(field.settings.dateFormat, value) : ''
+};
+
+fieldTypes['birthday'] = {
+    validate: field => {
+        enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
+    },
+    addColumn: (table, name) => table.dateTime(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeBirthday' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
+    forHbs: (field, value) => formatBirthday(field.settings.dateFormat, value),
+    parsePostValue: (field, value) => parseBirthday(field.settings.dateFormat, value),
+    render: (field, value) => value !== null ? formatBirthday(field.settings.dateFormat, value) : ''
+};
+
+const groupedTypes = Object.keys(fieldTypes).filter(key => fieldTypes[key].grouped);
+
+function getFieldType(type) {
+    return fieldTypes[type];
+}
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function getById(context, listId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewFields');
+
+        const entity = await tx('custom_fields').where({list: listId, id}).first();
+
+        entity.settings = JSON.parse(entity.settings);
+
+        const orderFields = {
+            order_list: 'orderListBefore',
+            order_subscribe: 'orderSubscribeBefore',
+            order_manage: 'orderManageBefore'
+        };
+
+        for (const key in orderFields) {
+            if (entity[key] !== null) {
+                const orderIdRow = await tx('custom_fields').where('list', listId).where(key, '>', entity[key]).orderBy(key, 'asc').select(['id']).first();
+                if (orderIdRow) {
+                    entity[orderFields[key]] = orderIdRow.id;
+                } else {
+                    entity[orderFields[key]] = 'end';
+                }
+            } else {
+                entity[orderFields[key]] = 'none';
+            }
+        }
+
+        return entity;
+    });
+}
+
+async function listTx(tx, listId) {
+    return await tx('custom_fields').where({list: listId}).select(['id', 'name', 'type', 'key', 'column', 'settings', 'group', 'default_value', 'order_list', 'order_subscribe', 'order_manage']).orderBy(knex.raw('-order_list'), 'desc').orderBy('id', 'asc');
+}
+
+async function list(context, listId) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewFields']);
+        return await listTx(tx, listId);
+    });
+}
+
+async function listGroupedTx(tx, listId) {
+    const flds = await listTx(tx, listId);
+
+    const fldsById = {};
+    for (const fld of flds) {
+        fld.settings = JSON.parse(fld.settings);
+
+        fldsById[fld.id] = fld;
+
+        if (fieldTypes[fld.type].grouped) {
+            fld.settings.options = [];
+            fld.groupedOptions = {};
+        }
+    }
+
+    for (const fld of flds) {
+        if (fld.group) {
+            const group = fldsById[fld.group];
+            group.settings.options.push({ key: fld.column, label: fld.name });
+            group.groupedOptions[fld.column] = fld;
+        }
+    }
+
+    const groupedFlds = flds.filter(fld => !fld.group);
+
+    for (const fld of flds) {
+        delete fld.group;
+    }
+
+    return groupedFlds;
+}
+
+async function listGrouped(context, listId) {
+    return await knex.transaction(async tx => {
+        // It may seem odd why there is not 'viewFields' here. Simply, at this point this function is needed only in managing subscriptions.
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['manageSubscriptions']);
+        return await listGroupedTx(tx, listId);
+    });
+}
+
+async function listByOrderListTx(tx, listId, extraColumns = []) {
+    return await tx('custom_fields').where({list: listId}).whereNotNull('order_list').select(['name', 'type', ...extraColumns]).orderBy('order_list', 'asc');
+}
+
+async function listDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewFields');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('custom_fields')
+
+                // This self join is to provide 'option' fields a reference to their parent grouped field. If the field is not an option, it refers to itself
+                // All this is to show options always below their group parent
+                .innerJoin('custom_fields AS parent_fields', function() {
+                    this.on(function() {
+                        this.on('custom_fields.type', '=', knex.raw('?', ['option']))
+                            .on('custom_fields.group', '=', 'parent_fields.id');
+                    }).orOn(function() {
+                        this.on('custom_fields.type', '<>', knex.raw('?', ['option']))
+                            .on('custom_fields.id', '=', 'parent_fields.id');
+                    });
+                })
+                .where('custom_fields.list', listId),
+            [ 'custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list' ],
+            {
+                orderByBuilder: (builder, orderColumn, orderDir) => {
+                    // We use here parent_fields to keep options always below their parent group
+                    if (orderColumn === 'custom_fields.order_list') {
+                        builder
+                            .orderBy(knex.raw('-parent_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
+                            .orderBy('parent_fields.name', orderDir)
+                            .orderBy(knex.raw('custom_fields.type = "option"'), 'asc')
+                    } else {
+                        const parentColumn = orderColumn.replace(/^custom_fields/, 'parent_fields');
+                        builder
+                            .orderBy(parentColumn, orderDir)
+                            .orderBy('parent_fields.name', orderDir)
+                            .orderBy(knex.raw('custom_fields.type = "option"'), 'asc');
+                    }
+                }
+            }
+        );
+    });
+}
+
+async function listGroupedDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewFields');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('custom_fields')
+                .where('custom_fields.list', listId)
+                .whereIn('custom_fields.type', groupedTypes),
+            ['custom_fields.id', 'custom_fields.name', 'custom_fields.type', 'custom_fields.key', 'custom_fields.order_list'],
+            {
+                orderByBuilder: (builder, orderColumn, orderDir) => {
+                    if (orderColumn === 'custom_fields.order_list') {
+                        builder
+                            .orderBy(knex.raw('-custom_fields.order_list'), orderDir === 'asc' ? 'desc' : 'asc') // This is MySQL speciality. It sorts the rows in ascending order with NULL values coming last
+                            .orderBy('custom_fields.name', orderDir);
+                    } else {
+                        builder
+                            .orderBy(orderColumn, orderDir)
+                            .orderBy('custom_fields.name', orderDir);
+                    }
+                }
+            }
+        );
+    });
+}
+
+async function serverValidate(context, listId, data) {
+    return await knex.transaction(async tx => {
+        const result = {};
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+        if (data.key) {
+            const existingKeyQuery = tx('custom_fields').where({
+                list: listId,
+                key: data.key
+            });
+
+            if (data.id) {
+                existingKeyQuery.whereNot('id', data.id);
+            }
+
+            const existingKey = await existingKeyQuery.first();
+            result.key = {
+                exists: !!existingKey
+            };
+        }
+
+        return result;
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, entity, isCreate) {
+    enforce(entity.type === 'option' || !entity.group, 'Only option may have a group assigned');
+    enforce(entity.type !== 'option' || entity.group, 'Option must have a group assigned.');
+    enforce(entity.type !== 'option' || (entity.orderListBefore === 'none' && entity.orderSubscribeBefore === 'none' && entity.orderManageBefore === 'none'), 'Option cannot be made visible');
+
+    enforce(!entity.group || await tx('custom_fields').where({list: listId, id: entity.group}).first(), 'Group field does not exist');
+    enforce(entity.name, 'Name must be present');
+
+    const fieldType = fieldTypes[entity.type];
+    enforce(fieldType, 'Unknown field type');
+
+    const validateErrs = fieldType.validate(entity);
+
+    enforce(validators.mergeTagValid(entity.key), 'Merge tag is not valid.');
+
+    const existingWithKeyQuery = tx('custom_fields').where({
+        list: listId,
+        key: entity.key
+    });
+    if (!isCreate) {
+        existingWithKeyQuery.whereNot('id', entity.id);
+    }
+    const existingWithKey = await existingWithKeyQuery.first();
+    if (existingWithKey) {
+        throw new interoperableErrors.DuplicitKeyError();
+    }
+
+    entity.settings = JSON.stringify(entity.settings);
+}
+
+async function _sortIn(tx, listId, entityId, orderListBefore, orderSubscribeBefore, orderManageBefore) {
+    const flds = await tx('custom_fields').where('list', listId).whereNot('id', entityId);
+
+    const order = {};
+    for (const row of flds) {
+        order[row.id] = {
+            order_list: null,
+            order_subscribe: null,
+            order_manage: null
+        };
+    }
+
+    order[entityId] = {
+        order_list: null,
+        order_subscribe: null,
+        order_manage: null
+    };
+
+    function computeOrder(fldName, sortInBefore) {
+        flds.sort((x, y) => x[fldName] - y[fldName]);
+        const ids = flds.filter(x => x[fldName] !== null).map(x => x.id);
+
+        let sortedIn = false;
+        let idx = 1;
+        for (const id of ids) {
+            if (sortInBefore === id) {
+                order[entityId][fldName] = idx;
+                sortedIn = true;
+                idx += 1;
+            }
+
+            order[id][fldName] = idx;
+            idx += 1;
+        }
+
+        if (!sortedIn && sortInBefore !== 'none') {
+            order[entityId][fldName] = idx;
+        }
+    }
+
+    computeOrder('order_list', orderListBefore);
+    computeOrder('order_subscribe', orderSubscribeBefore);
+    computeOrder('order_manage', orderManageBefore);
+
+    for (const id in order) {
+        await tx('custom_fields').where({list: listId, id}).update(order[id]);
+    }
+}
+
+async function create(context, listId, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+        await _validateAndPreprocess(tx, listId, entity, true);
+
+        const fieldType = fieldTypes[entity.type];
+
+        let columnName;
+        if (!fieldType.grouped) {
+            columnName = ('custom_' + slugify(entity.name, '_') + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '');
+        }
+
+        const filteredEntity = filterObject(entity, allowedKeysCreate);
+        filteredEntity.list = listId;
+        filteredEntity.column = columnName;
+
+        const ids = await tx('custom_fields').insert(filteredEntity);
+        const id = ids[0];
+
+        await _sortIn(tx, listId, id, entity.orderListBefore, entity.orderSubscribeBefore, entity.orderManageBefore);
+
+        if (columnName) {
+            await knex.schema.table('subscription__' + listId, table => {
+                fieldType.addColumn(table, columnName);
+                if (fieldType.indexed) {
+                    table.index(columnName);
+                }
+            });
+        }
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, listId, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+        const existing = await tx('custom_fields').where({list: listId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.settings = JSON.parse(existing.settings);
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        enforce(entity.type === existing.type, 'Field type cannot be changed');
+        await _validateAndPreprocess(tx, listId, entity, false);
+
+        await tx('custom_fields').where({list: listId, id: entity.id}).update(filterObject(entity, allowedKeysUpdate));
+        await _sortIn(tx, listId, entity.id, entity.orderListBefore, entity.orderSubscribeBefore, entity.orderManageBefore);
+    });
+}
+
+async function removeTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageFields');
+
+    const existing = await tx('custom_fields').where({list: listId, id: id}).first();
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    const fieldType = fieldTypes[existing.type];
+
+    await tx('custom_fields').where({list: listId, id}).del();
+
+    if (fieldType.grouped) {
+        await tx('custom_fields').where({list: listId, group: id}).del();
+
+    } else {
+        await knex.schema.table('subscription__' + listId, table => {
+            table.dropColumn(existing.column);
+        });
+
+        await segments.removeRulesByColumnTx(tx, context, listId, existing.column);
+    }
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, listId, id);
+    });
+}
+
+async function removeAllByListIdTx(tx, context, listId) {
+    const entities = await tx('custom_fields').where('list', listId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, listId, entity.id);
+    }
+}
+
+function forHbsWithFieldsGrouped(fieldsGrouped, subscription) { // assumes grouped subscription
+    const customFields = [{
+        name: 'Email Address',
+        column: 'email',
+        key: 'EMAIL',
+        typeSubscriptionEmail: true,
+        value: subscription ? subscription.email : '',
+        order_subscribe: -1,
+        order_manage: -1
+    }];
+
+    for (const fld of fieldsGrouped) {
+        const type = fieldTypes[fld.type];
+        const fldCol = getFieldColumn(fld);
+
+        const entry = {
+            name: fld.name,
+            key: fld.key,
+            [type.getHbsType(fld)]: true,
+            order_subscribe: fld.order_subscribe,
+            order_manage: fld.order_manage,
+        };
+
+        if (!type.grouped && !type.enumerated) {
+            // subscription[fldCol] may not exists because we are getting the data from "fromPost"
+            entry.value = (subscription ? type.forHbs(fld, subscription[fldCol]) : null) || '';
+
+        } else if (type.grouped) {
+            const options = [];
+            const value = (subscription ? subscription[fldCol] : null) || (type.cardinality === Cardinality.SINGLE ? null : []);
+
+            for (const optCol in fld.groupedOptions) {
+                const opt = fld.groupedOptions[optCol];
+
+                let isEnabled;
+                if (type.cardinality === Cardinality.SINGLE) {
+                    isEnabled = value === opt.column;
+                } else {
+                    isEnabled = value.includes(opt.column);
+                }
+
+                options.push({
+                    key: opt.key,
+                    name: opt.name,
+                    value: isEnabled
+                });
+            }
+
+            entry.options = options;
+
+        } else if (type.enumerated) {
+            const options = [];
+            const value = (subscription ? subscription[fldCol] : null) || null;
+
+            for (const opt of fld.settings.options) {
+                options.push({
+                    key: opt.key,
+                    name: opt.label,
+                    value: value === opt.key
+                });
+            }
+
+            entry.options = options;
+
+        }
+
+        customFields.push(entry);
+    }
+
+    return customFields;
+}
+
+// Returns an array that can be used for rendering by Handlebars
+async function forHbs(context, listId, subscription) { // assumes grouped subscription
+    const flds = await listGrouped(context, listId);
+    return forHbsWithFieldsGrouped(flds, subscription);
+}
+
+function getMergeTags(fieldsGrouped, subscription, extraTags = {}) { // assumes grouped subscription
+    const mergeTags = {
+        'EMAIL': subscription.email,
+        ...getMergeTagsForBases(getTrustedUrl(), getSandboxUrl(), getPublicUrl()),
+        ...extraTags
+    };
+
+    for (const fld of fieldsGrouped) {
+        const type = fieldTypes[fld.type];
+        const fldCol = getFieldColumn(fld);
+
+        mergeTags[fld.key] = type.render(fld, subscription[fldCol]);
+    }
+
+    return mergeTags;
+}
+
+
+// Converts subscription data received via (1) POST request from subscription form, (2) via subscribe request to API v1 to subscription structure supported by subscriptions model,
+// or (3) from import.
+// If a field is not specified in the POST data, it is also omitted in the returned subscription
+function _fromText(listId, data, flds, isGrouped, keyName, singleCardUsesKeyName) {
+
+    const subscription = {};
+
+    if (isGrouped) {
+        for (const fld of flds) {
+            const fldKey = fld[keyName];
+            if (fldKey && fldKey in data) {
+                const type = fieldTypes[fld.type];
+                const fldCol = getFieldColumn(fld);
+
+                let value = null;
+
+                if (!type.grouped && !type.enumerated) {
+                    value = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
+
+                } else if (type.grouped) {
+                    if (type.cardinality === Cardinality.SINGLE) {
+                        for (const optCol in fld.groupedOptions) {
+                            const opt = fld.groupedOptions[optCol];
+                            const optKey = opt[keyName];
+
+                            // This handles two different formats for grouped dropdowns and radios.
+                            // The first part of the condition handles the POST requests from the subscription form, while the
+                            // second part handles the subscribe request to API v1
+                            if (singleCardUsesKeyName) {
+                                if (data[fldKey] === optKey) {
+                                    value = opt.column
+                                }
+                            } else {
+                                const optType = fieldTypes[opt.type];
+                                const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
+                                if (optValue) {
+                                    value = opt.column
+                                }
+                            }
+                        }
+                    } else {
+                        value = [];
+
+                        for (const optCol in fld.groupedOptions) {
+                            const opt = fld.groupedOptions[optCol];
+                            const optKey = opt[keyName];
+                            const optType = fieldTypes[opt.type];
+                            const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
+
+                            if (optValue) {
+                                value.push(opt.column);
+                            }
+                        }
+                    }
+
+                } else if (type.enumerated) {
+                    value = data[fldKey];
+                }
+
+                subscription[fldCol] = value;
+            }
+        }
+
+    } else {
+        for (const fld of flds) {
+            const fldKey = fld[keyName];
+            if (fldKey && fldKey in data) {
+                const type = fieldTypes[fld.type];
+                const fldCol = getFieldColumn(fld);
+
+                subscription[fldCol] = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
+            }
+        }
+    }
+
+    return subscription;
+}
+
+async function fromPost(context, listId, data) { // assumes grouped subscription and indexation by merge key
+    const flds = await listGrouped(context, listId);
+    return _fromText(listId, data, flds, true, 'key', true);
+}
+
+async function fromAPI(context, listId, data) { // assumes grouped subscription and indexation by merge key
+    const flds = await listGrouped(context, listId);
+    return _fromText(listId, data, flds, true, 'key', false);
+}
+
+function fromImport(listId, flds, data) { // assumes ungrouped subscription and indexation by column
+    return _fromText(listId, data, flds, true, 'column', false);
+}
+
+module.exports.Cardinality = Cardinality;
+module.exports.getFieldType = getFieldType;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.list = list;
+module.exports.listTx = listTx;
+module.exports.listGrouped = listGrouped;
+module.exports.listGroupedTx = listGroupedTx;
+module.exports.listByOrderListTx = listByOrderListTx;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listGroupedDTAjax = listGroupedDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
+module.exports.serverValidate = serverValidate;
+module.exports.forHbs = forHbs;
+module.exports.forHbsWithFieldsGrouped = forHbsWithFieldsGrouped;
+module.exports.fromPost = fromPost;
+module.exports.fromAPI = fromAPI;
+module.exports.fromImport = fromImport;
+module.exports.getMergeTags = getMergeTags;

server/models/files.js

@@ -0,0 +1,342 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const shares = require('./shares');
+const fs = require('fs-extra-promise');
+const path = require('path');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const entitySettings = require('../lib/entity-settings');
+const {getPublicUrl} = require('../lib/urls');
+
+const crypto = require('crypto');
+const bluebird = require('bluebird');
+const cryptoPseudoRandomBytes = bluebird.promisify(crypto.pseudoRandomBytes);
+
+const entityTypes = entitySettings.getEntityTypes();
+
+const filesDir = path.join(__dirname, '..', 'files');
+
+const ReplacementBehavior = entitySettings.ReplacementBehavior;
+
+function enforceTypePermitted(type, subType) {
+    enforce(type in entityTypes && entityTypes[type].files && entityTypes[type].files[subType], `File type ${type}:${subType} does not exist`);
+}
+
+function getFilePath(type, subType, entityId, filename) {
+    return path.join(path.join(filesDir, type, subType, entityId.toString()), filename);
+}
+
+function getFileUrl(context, type, subType, entityId, filename) {
+    return getPublicUrl(`files/${type}/${subType}/${entityId}/${filename}`, context)
+}
+
+function getFilesTable(type, subType) {
+    return entityTypes[type].files[subType].table;
+}
+
+function getFilesPermission(type, subType, operation) {
+    return entityTypes[type].files[subType].permissions[operation];
+}
+
+async function listDTAjax(context, type, subType, entityId, params) {
+    enforceTypePermitted(type, subType);
+    await shares.enforceEntityPermission(context, type, entityId, getFilesPermission(type, subType, 'view'));
+    return await dtHelpers.ajaxList(
+        params,
+        builder => builder.from(getFilesTable(type, subType)).where({entity: entityId}),
+        ['id', 'originalname', 'filename', 'size', 'created']
+    );
+}
+
+async function listTx(tx, context, type, subType, entityId) {
+    enforceTypePermitted(type, subType);
+    await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
+    return await tx(getFilesTable(type, subType)).where({entity: entityId}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc');
+}
+
+async function list(context, type, subType, entityId) {
+    return await knex.transaction(async tx => {
+        return await listTx(tx, context, type, subType, entityId);
+    });
+}
+
+async function getFileById(context, type, subType, id) {
+    enforceTypePermitted(type, subType);
+    const file = await knex.transaction(async tx => {
+        const file = await tx(getFilesTable(type, subType)).where('id', id).first();
+        await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'view'));
+        return file;
+    });
+
+    if (!file) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    return {
+        mimetype: file.mimetype,
+        name: file.originalname,
+        path: getFilePath(type, subType, file.entity, file.filename)
+    };
+}
+
+async function _getFileBy(context, type, subType, entityId, key, value) {
+    enforceTypePermitted(type, subType);
+    const file = await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
+        const file = await tx(getFilesTable(type, subType)).where({entity: entityId, [key]: value}).first();
+        return file;
+    });
+
+    if (!file) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    return {
+        mimetype: file.mimetype,
+        name: file.originalname,
+        path: getFilePath(type, subType, file.entity, file.filename)
+    };
+}
+
+async function getFileByOriginalName(context, type, subType, entityId, name) {
+    return await _getFileBy(context, type, subType, entityId, 'originalname', name)
+}
+
+async function getFileByFilename(context, type, subType, entityId, name) {
+    return await _getFileBy(context, type, subType, entityId, 'filename', name)
+}
+
+async function getFileByUrl(context, url) {
+    const urlPrefix = getPublicUrl('files/', context);
+    if (url.startsWith(urlPrefix)) {
+        const path = url.substring(urlPrefix.length);
+        const pathElem = path.split('/');
+
+        if (pathElem.length !== 4) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const type = pathElem[0];
+        const subType = pathElem[1];
+        const entityId = Number.parseInt(pathElem[2]);
+
+        if (Number.isNaN(entityId)) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const name = pathElem[3];
+
+        return await getFileByFilename(context, type, subType, entityId, name);
+    } else {
+        throw new interoperableErrors.NotFoundError();
+    }
+}
+
+// Adds files to an entity. The source data can be either a file (then it's path is contained in file.path) or in-memory data (then it's content is in file.data).
+async function createFiles(context, type, subType, entityId, files, replacementBehavior, transformResponseFn) {
+    enforceTypePermitted(type, subType);
+    if (files.length == 0) {
+        // No files uploaded
+        return {uploaded: 0};
+    }
+
+    if (!replacementBehavior) {
+        replacementBehavior = entityTypes[type].files[subType].defaultReplacementBehavior;
+    }
+
+    const fileEntities = [];
+    const filesToMove = [];
+    const ignoredFiles = [];
+    const removedFiles = [];
+    const filesRet = [];
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage'));
+
+        const existingNamesRows = await tx(getFilesTable(type, subType)).where('entity', entityId).select(['id', 'filename', 'originalname']);
+
+        const existingNameSet = new Set();
+        for (const row of existingNamesRows) {
+            existingNameSet.add(row.originalname);
+        }
+
+        // The processedNameSet holds originalnames of entries which have been already processed in the upload batch. It prevents uploading two files with the same originalname
+        const processedNameSet = new Set();
+
+
+        // Create entities for files
+        for (const file of files) {
+            const parsedOriginalName = path.parse(file.originalname);
+            let originalName = parsedOriginalName.base;
+
+            if (!file.filename) {
+                // This is taken from multer/storage/disk.js and adapted for async/await
+                file.filename = (await cryptoPseudoRandomBytes(16)).toString('hex');
+            }
+
+            if (replacementBehavior === ReplacementBehavior.RENAME) {
+                let suffix = 1;
+                while (existingNameSet.has(originalName) || processedNameSet.has(originalName)) {
+                    originalName = parsedOriginalName.name + '-' + suffix + parsedOriginalName.ext;
+                    suffix++;
+                }
+            }
+
+            if (replacementBehavior === ReplacementBehavior.NONE && (existingNameSet.has(originalName) || processedNameSet.has(originalName))) {
+                // The file has an original name same as another file in the same upload batch or it has an original name same as another already existing file
+                ignoredFiles.push(file);
+
+            } else {
+                filesToMove.push(file);
+
+                fileEntities.push({
+                    entity: entityId,
+                    filename: file.filename,
+                    originalname: originalName,
+                    mimetype: file.mimetype,
+                    size: file.size
+                });
+
+                const filesRetEntry = {
+                    name: file.filename,
+                    originalName: originalName,
+                    size: file.size,
+                    type: file.mimetype
+                };
+
+                filesRetEntry.url = getFileUrl(context, type, subType, entityId, file.filename);
+
+                if (file.mimetype.startsWith('image/')) {
+                    filesRetEntry.thumbnailUrl = getFileUrl(context, type, subType, entityId, file.filename); // TODO - use smaller thumbnails,
+                }
+
+                filesRet.push(filesRetEntry);
+            }
+
+            processedNameSet.add(originalName);
+        }
+
+        if (replacementBehavior === ReplacementBehavior.REPLACE) {
+            const idsToRemove = [];
+            for (const row of existingNamesRows) {
+                if (processedNameSet.has(row.originalname)) {
+                    removedFiles.push(row);
+                    idsToRemove.push(row.id);
+                }
+            }
+
+            await tx(getFilesTable(type, subType)).where('entity', entityId).whereIn('id', idsToRemove).del();
+        }
+
+        if (fileEntities) {
+            await tx(getFilesTable(type, subType)).insert(fileEntities);
+        }
+    });
+
+    // Move new files from upload directory to files directory
+    for (const file of filesToMove) {
+        const filePath = getFilePath(type, subType, entityId, file.filename);
+
+        if (file.path) {
+            // The names should be unique, so overwrite is disabled
+            // The directory is created if it does not exist
+            // Empty options argument is passed, otherwise fails
+            await fs.moveAsync(file.path, filePath, {});
+        } else if (file.data) {
+            await fs.outputFile(filePath, file.data);
+        }
+    }
+    // Remove replaced files from files directory
+    for (const file of removedFiles) {
+        const filePath = getFilePath(type, subType, entityId, file.filename);
+        await fs.removeAsync(filePath);
+    }
+    // Remove ignored files from upload directory
+    for (const file of ignoredFiles) {
+        if (file.path) {
+            await fs.removeAsync(file.path);
+        }
+    }
+
+    const resp = {
+        uploaded: files.length,
+        added: fileEntities.length - removedFiles.length,
+        replaced: removedFiles.length,
+        ignored: ignoredFiles.length,
+        files: filesRet
+    };
+
+    if (transformResponseFn) {
+        return transformResponseFn(resp);
+    } else {
+        return resp;
+    }
+}
+
+async function removeFile(context, type, subType, id) {
+    enforceTypePermitted(type, subType);
+
+    const file = await knex.transaction(async tx => {
+        const file = await tx(getFilesTable(type, subType)).where('id', id).select('entity', 'filename').first();
+        await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'manage'));
+        await tx(getFilesTable(type, subType)).where('id', id).del();
+        return {filename: file.filename, entity: file.entity};
+    });
+
+    const filePath = getFilePath(type, subType, file.entity, file.filename);
+    await fs.removeAsync(filePath);
+}
+
+async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toType, toSubType, toEntityId) {
+    enforceTypePermitted(fromType, fromSubType);
+    await shares.enforceEntityPermissionTx(tx, context, fromType, fromEntityId, getFilesPermission(fromType, fromSubType, 'view'));
+
+    enforceTypePermitted(toType, toSubType);
+    await shares.enforceEntityPermissionTx(tx, context, toType, toEntityId, getFilesPermission(toType, toSubType, 'manage'));
+
+    const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId});
+    for (const row of rows) {
+        const fromFilePath = getFilePath(fromType, fromSubType, fromEntityId, row.filename);
+        const toFilePath = getFilePath(toType, toSubType, toEntityId, row.filename);
+        await fs.copyAsync(fromFilePath, toFilePath, {});
+
+        delete row.id;
+        row.entity = toEntityId;
+    }
+
+    if (rows.length > 0) {
+        await tx(getFilesTable(toType, toSubType)).insert(rows);
+    }
+}
+
+async function removeAllTx(tx, context, type, subType, entityId) {
+    enforceTypePermitted(type, subType);
+    await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage'));
+
+    const rows = await tx(getFilesTable(type, subType)).where({entity: entityId});
+    for (const row of rows) {
+        const filePath = getFilePath(type, subType, entityId, row.filename);
+        await fs.removeAsync(filePath);
+    }
+
+    await tx(getFilesTable(type, subType)).where('entity', entityId).del();
+}
+
+
+module.exports.filesDir = filesDir;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listTx = listTx;
+module.exports.list = list;
+module.exports.getFileById = getFileById;
+module.exports.getFileByFilename = getFileByFilename;
+module.exports.getFileByUrl = getFileByUrl;
+module.exports.getFileByOriginalName = getFileByOriginalName;
+module.exports.createFiles = createFiles;
+module.exports.removeFile = removeFile;
+module.exports.getFileUrl = getFileUrl;
+module.exports.getFilePath = getFilePath;
+module.exports.copyAllTx = copyAllTx;
+module.exports.removeAllTx = removeAllTx;
+module.exports.ReplacementBehavior = ReplacementBehavior;

server/models/forms.js

@@ -0,0 +1,278 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const bluebird = require('bluebird');
+const fs = require('fs-extra');
+const path = require('path');
+const mjml = require('mjml');
+const _ = require('../lib/translate')._;
+const lists = require('./lists');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const formAllowedKeys = new Set([
+    'name',
+    'description',
+    'layout',
+    'form_input_style',
+    'namespace'
+]);
+
+const allowedFormKeys = new Set([
+    'web_subscribe',
+    'web_confirm_subscription_notice',
+    'mail_confirm_subscription_html',
+    'mail_confirm_subscription_text',
+    'mail_already_subscribed_html',
+    'mail_already_subscribed_text',
+    'web_subscribed_notice',
+    'mail_subscription_confirmed_html',
+    'mail_subscription_confirmed_text',
+    'web_manage',
+    'web_manage_address',
+    'web_updated_notice',
+    'web_unsubscribe',
+    'web_confirm_unsubscription_notice',
+    'mail_confirm_unsubscription_html',
+    'mail_confirm_unsubscription_text',
+    'mail_confirm_address_change_html',
+    'mail_confirm_address_change_text',
+    'web_unsubscribed_notice',
+    'mail_unsubscription_confirmed_html',
+    'mail_unsubscription_confirmed_text',
+    'web_manual_unsubscribe_notice'
+]);
+
+const hashKeys = new Set([...formAllowedKeys, ...allowedFormKeys]);
+
+const allowedKeysServerValidate = new Set(['layout', ...allowedFormKeys]);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'customForm', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('custom_forms')
+            .innerJoin('namespaces', 'namespaces.id', 'custom_forms.namespace'),
+        ['custom_forms.id', 'custom_forms.name', 'custom_forms.description', 'namespaces.name']
+    );
+}
+
+
+async function _getById(tx, id) {
+    const entity = await tx('custom_forms').where('id', id).first();
+
+    if (!entity) {
+        throw interoperableErrors.NotFoundError();
+    }
+
+    const forms = await tx('custom_forms_data').where('form', id).select(['data_key', 'data_value']);
+
+    for (const form of forms) {
+        entity[form.data_key] = form.data_value;
+    }
+
+    return entity;
+}
+
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'customForm', id, 'view');
+        const entity = await _getById(tx, id);
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'customForm', id);
+        return entity;
+    });
+}
+
+
+async function serverValidate(context, data) {
+    const result = {};
+
+    const form = filterObject(data, allowedKeysServerValidate);
+    const errs = checkForMjmlErrors(form);
+
+    for (const key in form) {
+        result[key] = {};
+        if (errs[key]) {
+            result[key].errors = errs[key];
+        }
+    }
+
+    return result;
+}
+
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createCustomForm');
+
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        const form = filterObject(entity, allowedFormKeys);
+        enforce(!Object.keys(checkForMjmlErrors(form)).length, 'Error(s) in form templates');
+
+        const ids = await tx('custom_forms').insert(filterObject(entity, formAllowedKeys));
+        const id = ids[0];
+
+        for (const formKey in form) {
+            await tx('custom_forms_data').insert({
+                form: id,
+                data_key: formKey,
+                data_value: form[formKey]
+            })
+        }
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'customForm', entityId: id });
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'customForm', entity.id, 'edit');
+
+        const existing = await _getById(tx, entity.id);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'customForm', 'createCustomForm', 'delete');
+
+        const form = filterObject(entity, allowedFormKeys);
+        enforce(!Object.keys(checkForMjmlErrors(form)).length, 'Error(s) in form templates');
+
+        await tx('custom_forms').where('id', entity.id).update(filterObject(entity, formAllowedKeys));
+
+        for (const formKey in form) {
+            await tx('custom_forms_data').update({
+                data_value: form[formKey]
+            }).where({
+                form: entity.id,
+                data_key: formKey
+            });
+        }
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'customForm', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'customForm', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            { entityTypeId: 'list', column: 'default_form' }
+        ]);
+
+        await tx('custom_forms_data').where('form', id).del();
+        await tx('custom_forms').where('id', id).del();
+    });
+}
+
+
+// FIXME - add the ability of having multiple language variant of the same custom form
+async function getDefaultCustomFormValues() {
+    const basePath = path.join(__dirname, '..');
+
+    async function getContents(fileName) {
+        try {
+            const template = await fs.readFile(path.join(basePath, fileName), 'utf8');
+        } catch (err) {
+            return false;
+        }
+    }
+
+    const form = {};
+
+    for (const key of allowedFormKeys) {
+        const base = 'views/subscription/' + key.replace(/_/g, '-');
+        if (key.startsWith('mail') || key.startsWith('web')) {
+            form[key] = await getContents(base + '.mjml.hbs') || await getContents(base + '.hbs') || '';
+        }
+    }
+
+    form.layout = await getContents('views/subscription/layout.mjml.hbs') || '';
+    form.form_input_style = await getContents('static/subscription/form-input-style.css') || '@import url(/subscription/form-input-style.css);';
+
+    return form;
+}
+
+
+function checkForMjmlErrors(form) {
+    let testLayout = '<mjml><mj-body><mj-container>{{{body}}}</mj-container></mj-body></mjml>';
+
+    let hasMjmlError = (template, layout = testLayout) => {
+        let source = layout.replace(/\{\{\{body\}\}\}/g, template);
+        let compiled;
+
+        try {
+            compiled = mjml(source);
+        } catch (err) {
+            return err;
+        }
+
+        return compiled.errors;
+    };
+
+
+    const errors = {};
+    for (const key in form) {
+        if (key.startsWith('mail_') || key.startsWith('web_')) {
+            const template = form[key];
+            const errs = hasMjmlError(template);
+
+            const msgs = errs.map(x => x.formattedMessage);
+            if (key === 'mail_confirm_html' && !template.includes('{{confirmUrl}}')) {
+                msgs.push('Missing {{confirmUrl}}');
+            }
+
+            if (msgs.length) {
+                errors[key] = msgs;
+            }
+
+        } else if (key === 'layout') {
+            const layout = form[key];
+            const errs = hasMjmlError('', layout);
+
+            let msgs;
+            if (Array.isArray(errs)) {
+                msgs = errs.map(x => x.formattedMessage)
+            } else {
+                msgs = [ errs.message ];
+            }
+
+            if (!layout.includes('{{{body}}}')) {
+                msgs.push(`{{{body}}} not found`);
+            }
+
+            if (msgs.length) {
+                errors[key] = msgs;
+            }
+        }
+    }
+
+    return errors;
+}
+
+module.exports.listDTAjax = listDTAjax;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.getDefaultCustomFormValues = getDefaultCustomFormValues;
+module.exports.serverValidate = serverValidate;

server/models/import-runs.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+
+async function getById(context, listId, importId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        const entity = await tx('import_runs')
+            .innerJoin('imports', 'import_runs.import', 'imports.id')
+            .where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': id})
+            .select('import_runs.id', 'import_runs.import', 'import_runs.status', 'import_runs.new',
+                'import_runs.failed', 'import_runs.processed', 'import_runs.error', 'import_runs.created', 'import_runs.finished')
+            .first();
+
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, listId, importId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('import_runs')
+                .innerJoin('imports', 'import_runs.import', 'imports.id')
+                .where({'imports.list': listId, 'imports.id': importId})
+                .orderBy('import_runs.id', 'desc'),
+            [ 'import_runs.id', 'import_runs.created', 'import_runs.finished', 'import_runs.status', 'import_runs.processed', 'import_runs.new', 'import_runs.failed']
+        );
+    });
+}
+
+async function listFailedDTAjax(context, listId, importId, importRunId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('import_failed')
+                .innerJoin('import_runs', 'import_failed.run', 'import_runs.id')
+                .innerJoin('imports', 'import_runs.import', 'imports.id')
+                .where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': importRunId})
+                .orderBy('import_failed.source_id', 'asc'),
+            [ 'import_failed.id', 'import_failed.source_id', 'import_failed.email', 'import_failed.reason']
+        );
+    });
+}
+
+
+
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listFailedDTAjax = listFailedDTAjax;

server/models/imports.js

@@ -0,0 +1,250 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const {ImportSource, MappingType, ImportStatus, RunStatus, prepFinished, prepFinishedAndNotInProgress, runInProgress} = require('../../shared/imports');
+const fs = require('fs-extra-promise');
+const path = require('path');
+const importer = require('../lib/importer');
+
+const filesDir = path.join(__dirname, '..', 'files', 'imports');
+
+const allowedKeysCreate = new Set(['name', 'description', 'source', 'settings']);
+const allowedKeysUpdate = new Set(['name', 'description', 'mapping_type', 'mapping']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeysUpdate));
+}
+
+async function getById(context, listId, id, withSampleRow = false) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        entity.settings = JSON.parse(entity.settings);
+        entity.mapping = JSON.parse(entity.mapping);
+
+        if (withSampleRow && prepFinished(entity.status)) {
+            if (entity.source === ImportSource.CSV_FILE) {
+                const importTable = 'import_file__' + id;
+
+                const row = await tx(importTable).first();
+                delete row.id;
+
+                entity.sampleRow = row;
+            }
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('imports')
+                .where('imports.list', listId),
+            [ 'imports.id', 'imports.name', 'imports.description', 'imports.source', 'imports.status', 'imports.last_run' ]
+        );
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, entity, isCreate) {
+    if (isCreate) {
+        enforce(Number.isInteger(entity.source));
+        enforce(entity.source >= ImportSource.MIN && entity.source <= ImportSource.MAX, 'Invalid import source');
+
+        entity.settings = entity.settings || {};
+
+        if (entity.source === ImportSource.CSV_FILE) {
+            entity.settings.csv = entity.settings.csv || {};
+            enforce(entity.settings.csv.delimiter && entity.settings.csv.delimiter.trim(), 'CSV delimiter must not be empty');
+        }
+
+    } else {
+        enforce(Number.isInteger(entity.mapping_type));
+        enforce(entity.mapping_type >= MappingType.MIN && entity.mapping_type <= MappingType.MAX, 'Invalid mapping type');
+
+        entity.mapping = entity.mapping || { settings: {}, fields: {} };
+    }
+}
+
+async function create(context, listId, entity, files) {
+    const res = await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        await _validateAndPreprocess(tx, listId, entity, true);
+
+        if (entity.source === ImportSource.CSV_FILE) {
+            enforce(files.csvFile, 'File must be included');
+            const csvFile = files.csvFile[0];
+            const filePath = path.join(filesDir, csvFile.filename);
+            await fs.moveAsync(csvFile.path, filePath, {});
+
+            entity.settings.csv = {
+                originalname: csvFile.originalname,
+                filename: csvFile.filename,
+                delimiter: entity.settings.csv.delimiter
+            };
+
+            entity.status = ImportStatus.PREP_SCHEDULED;
+        }
+
+        const filteredEntity = filterObject(entity, allowedKeysCreate);
+        filteredEntity.list = listId;
+        filteredEntity.settings = JSON.stringify(filteredEntity.settings);
+
+        filteredEntity.mapping_type = MappingType.BASIC_SUBSCRIBE; // This is not set in the create form. It can be changed in the update form.
+        filteredEntity.mapping = JSON.stringify({});
+
+        const ids = await tx('imports').insert(filteredEntity);
+        const id = ids[0];
+
+        return id;
+    });
+
+    importer.scheduleCheck();
+    return res;
+}
+
+async function updateWithConsistencyCheck(context, listId, entity) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const existing = await tx('imports').where({list: listId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.mapping = JSON.parse(existing.mapping);
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        enforce(prepFinished(existing.status), 'Cannot save updates until preparation is finished');
+
+        await _validateAndPreprocess(tx, listId, entity, false);
+
+        const filteredEntity = filterObject(entity, allowedKeysUpdate);
+        filteredEntity.mapping = JSON.stringify(filteredEntity.mapping);
+
+        await tx('imports').where({list: listId, id: entity.id}).update(filteredEntity);
+    });
+}
+
+async function removeTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+    const existing = await tx('imports').where({list: listId, id: id}).first();
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    existing.settings = JSON.parse(existing.settings);
+
+    const filePath = path.join(filesDir, existing.settings.csv.filename);
+    await fs.removeAsync(filePath);
+
+    const importTable = 'import_file__' + id;
+    await knex.schema.dropTableIfExists(importTable);
+
+    await tx('import_failed').whereIn('run', function() {this.from('import_runs').select('id').where('import', id)}).del();
+    await tx('import_runs').where('import', id).del();
+    await tx('imports').where({list: listId, id}).del();
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, listId, id);
+    });
+}
+
+async function removeAllByListIdTx(tx, context, listId) {
+    const entities = await tx('imports').where('list', listId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, listId, entity.id);
+    }
+}
+
+async function start(context, listId, id) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!prepFinishedAndNotInProgress(entity.status)) {
+            throw new interoperableErrors.InvalidStateError('Cannot start until preparation or run is finished');
+        }
+
+        await tx('imports').where({list: listId, id}).update({
+            status: ImportStatus.RUN_SCHEDULED
+        });
+
+        await tx('import_runs').insert({
+            import: id,
+            status: RunStatus.SCHEDULED,
+            mapping: entity.mapping
+        });
+    });
+
+    importer.scheduleCheck();
+}
+
+async function stop(context, listId, id) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!runInProgress(entity.status)) {
+            throw new interoperableErrors.InvalidStateError('No import is currently running');
+        }
+
+        await tx('imports').where({list: listId, id}).update({
+            status: ImportStatus.RUN_STOPPING
+        });
+
+        await tx('import_runs').where('import', id).whereIn('status', [RunStatus.SCHEDULED, RunStatus.RUNNING]).update({
+            status: RunStatus.STOPPING
+        });
+    });
+
+    importer.scheduleCheck();
+}
+
+
+module.exports.filesDir = filesDir;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
+module.exports.start = start;
+module.exports.stop = stop;

server/models/links.js

@@ -0,0 +1,180 @@
+'use strict';
+
+const log = require('../lib/log');
+const knex = require('../lib/knex');
+const dtHelpers = require('../lib/dt-helpers');
+const shares = require('./shares');
+const campaigns = require('./campaigns');
+const lists = require('./lists');
+const subscriptions = require('./subscriptions');
+const contextHelpers = require('../lib/context-helpers');
+const geoip = require('geoip-ultralight');
+const uaParser = require('device');
+const he = require('he');
+const { enforce } = require('../lib/helpers');
+const { getPublicUrl } = require('../lib/urls');
+const tools = require('../lib/tools');
+
+const LinkId = {
+    OPEN: -1,
+    GENERAL_CLICK: 0
+};
+
+async function resolve(linkCid) {
+    return await knex('links').where('cid', linkCid).select(['id', 'url']).first();
+}
+
+async function countLink(remoteIp, userAgent, campaignCid, listCid, subscriptionCid, linkId) {
+    await knex.transaction(async tx => {
+        const list = await lists.getByCidTx(tx, contextHelpers.getAdminContext(), listCid);
+        const campaign = await campaigns.getTrackingSettingsByCidTx(tx, campaignCid);
+        const subscription = await subscriptions.getByCidTx(tx, contextHelpers.getAdminContext(), list.id, subscriptionCid);
+
+        const country = geoip.lookupCountry(remoteIp) || null;
+        const device = uaParser(userAgent, { unknownUserAgentDeviceType: 'desktop', emptyUserAgentDeviceType: 'desktop' });
+        const now = new Date();
+
+        const _countLink = async (clickLinkId, incrementOnDup) => {
+            try {
+                const campaignLinksQry = knex('campaign_links')
+                    .insert({
+                        campaign: campaign.id,
+                        list: list.id,
+                        subscription: subscription.id,
+                        link: linkId,
+                        ip: remoteIp,
+                        device_type: device.type,
+                        country
+                    }).toSQL();
+
+                const campaignLinksQryResult = await tx.raw(campaignLinksQry.sql + (incrementOnDup ? ' ON DUPLICATE KEY UPDATE `count`=`count`+1' : ''), campaignLinksQry.bindings);
+
+                if (campaignLinksQryResult.affectedRows > 1) { // When using DUPLICATE KEY UPDATE, this means that the entry was already there
+                    return false;
+                }
+
+                return true;
+
+            } catch (err) {
+                if (err.code === 'ER_DUP_ENTRY') {
+                    return false;
+                }
+
+                throw err;
+            }
+        };
+
+
+
+        // Update opened and click timestamps
+        const latestUpdates = {};
+
+        if (!campaign.click_tracking_disabled && linkId > LinkId.GENERAL_CLICK) {
+            latestUpdates.latest_click = now;
+        }
+
+        if (!campaign.open_tracking_disabled) {
+            latestUpdates.latest_open = now;
+        }
+
+        if (latestUpdates.latest_click || latestUpdates.latest_open) {
+            await tx(subscriptions.getSubscriptionTableName(list.id)).update(latestUpdates).where('id', subscription.id);
+        }
+
+
+        // Update clicks
+        if (linkId > LinkId.GENERAL_CLICK && !campaign.click_tracking_disabled) {
+            if (await _countLink(linkId, true)) {
+                if (await _countLink(LinkId.GENERAL_CLICK, false)) {
+                    await tx('campaigns').increment('clicks').where('id', campaign.id);
+                }
+            }
+        }
+
+
+        // Update opens. We count a click as an open too.
+        if (!campaign.open_tracking_disabled) {
+            if (await _countLink(LinkId.OPEN, true)) {
+                await tx('campaigns').increment('opened').where('id', campaign.id);
+            }
+        }
+    });
+}
+
+async function addOrGet(campaignId, url) {
+    return await knex.transaction(async tx => {
+        const link = tx('links').select(['id', 'cid']).where({
+            campaign: campaignId,
+            url
+        }).first();
+
+        if (!link) {
+            let cid = shortid.generate();
+
+            const ids = tx('links').insert({
+                campaign: campaignId,
+                cid,
+                url
+            });
+
+            return {
+                id: ids[0],
+                cid
+            };
+        }
+    });
+}
+
+async function updateLinks(campaign, list, subscription, mergeTags, message) {
+    if ((campaign.open_tracking_disabled && campaign.click_tracking_disabled) || !message || !message.trim()) {
+        // tracking is disabled, do not modify the message
+        return message;
+    }
+
+    // insert tracking image
+    if (!campaign.open_tracking_disabled) {
+        let inserted = false;
+        const imgUrl = getPublicUrl(`/links/${campaign.cid}/${list.cid}/${subscription.cid}`);
+        const img = '<img src="' + imgUrl + '" width="1" height="1" alt="mt">';
+        message = message.replace(/<\/body\b/i, match => {
+            inserted = true;
+            return img + match;
+        });
+        if (!inserted) {
+            message = message + img;
+        }
+    }
+
+    if (!campaign.click_tracking_disabled) {
+        const re = /(<a[^>]* href\s*=\s*["']\s*)(http[^"'>\s]+)/gi;
+
+        const urlsToBeReplaced = new Set();
+
+        message.replace(re, (match, prefix, encodedUrl) => {
+            const url = he.decode(encodedUrl, {isAttributeValue: true});
+            urlsToBeReplaced.add(url);
+        });
+
+        const urls = new Map(); // url -> {id, cid} (as returned by add)
+        for (const url of urlsToBeReplaced) {
+            // url might include variables, need to rewrite those just as we do with message content
+            const expanedUrl = tools.formatMessage(campaign, list, subscription, mergeTags, url);
+            const link = await addOrGet(campaign.id, expanedUrl);
+            urls.set(url, link);
+        }
+
+        message = message.replace(re, (match, prefix, encodedUrl) => {
+            const url = he.decode(encodedUrl, {isAttributeValue: true});
+            const link = urls.get(url);
+            return prefix + (link ? getPublicUrl(`/links/${campaign.cid}/${list.cid}/${subscription.cid}/${link.cid}`) : url);
+        });
+    }
+
+    return message;
+}
+
+module.exports.LinkId = LinkId;
+module.exports.resolve = resolve;
+module.exports.countLink = countLink;
+module.exports.addOrGet = addOrGet;
+module.exports.updateLinks = updateLinks;

server/models/lists.js

@@ -0,0 +1,210 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const fields = require('./fields');
+const segments = require('./segments');
+const imports = require('./imports');
+const entitySettings = require('../lib/entity-settings');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const UnsubscriptionMode = require('../../shared/lists').UnsubscriptionMode;
+
+const allowedKeys = new Set(['name', 'description', 'default_form', 'public_subscribe', 'unsubscription_mode', 'contact_email', 'homepage', 'namespace', 'to_name', 'listunsubscribe_disabled']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+
+async function listDTAjax(context, params) {
+    const campaignEntityType = entitySettings.getEntityType('campaign');
+
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace'),
+        ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description', 'namespaces.name',
+            { query: builder =>
+                builder.from('campaigns')
+                    .innerJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
+                    .innerJoin('triggers', 'campaigns.id', 'triggers.campaign')
+                    .innerJoin(campaignEntityType.permissionsTable, 'campaigns.id', `${campaignEntityType.permissionsTable}.entity`)
+                    .whereRaw('campaign_lists.list = lists.id')
+                    .where(`${campaignEntityType.permissionsTable}.operation`, 'viewTriggers')
+                    .count()
+            }
+        ]
+    );
+}
+
+async function listWithSegmentByCampaignDTAjax(context, campaignId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('campaign_lists', 'campaign_lists.list', 'lists.id')
+            .leftJoin('segments', 'segments.id', 'campaign_lists.segment')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace')
+            .where('campaign_lists.campaign', campaignId)
+            .orderBy('campaign_lists.id', 'asc'),
+        ['lists.id', 'lists.name', 'lists.cid', 'namespaces.name', 'segments.name']
+    );
+}
+
+async function getByIdTx(tx, context, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', id, 'view');
+    const entity = await tx('lists').where('id', id).first();
+    return entity;
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        // note that permissions are not obtained here as this methods is used only with synthetic admin context
+        return await getByIdTx(tx, context, id);
+    });
+}
+
+async function getByIdWithListFields(context, id) {
+    return await knex.transaction(async tx => {
+        const entity = await getByIdTx(tx, context, id);
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'list', id);
+        entity.listFields = await fields.listByOrderListTx(tx, id);
+        return entity;
+    });
+}
+
+async function getByCidTx(tx, context, cid) {
+    const entity = await tx('lists').where('cid', cid).first();
+    if (!entity) {
+        shares.throwPermissionDenied();
+    }
+
+    await shares.enforceEntityPermissionTx(tx, context, 'list', entity.id, 'view');
+    return entity;
+}
+
+async function getByCid(context, cid) {
+    return await knex.transaction(async tx => {
+        return getByCidTx(tx, context, cid);
+    });
+}
+
+async function _validateAndPreprocess(tx, entity) {
+    await namespaceHelpers.validateEntity(tx, entity);
+    enforce(entity.unsubscription_mode >= UnsubscriptionMode.MIN && entity.unsubscription_mode <= UnsubscriptionMode.MAX, 'Unknown unsubscription mode');
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createList');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.cid = shortid.generate();
+
+        const ids = await tx('lists').insert(filteredEntity);
+        const id = ids[0];
+
+        await knex.schema.raw('CREATE TABLE `subscription__' + id + '` (\n' +
+            '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
+            '  `cid` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
+            '  `email` varchar(255) CHARACTER SET utf8 NOT NULL,\n' +
+            '  `hash_email` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
+            '  `source_email` int(10) unsigned,\n' + // This references imports if the source is an import, 0 means some import in version 1, NULL if the source is via subscription or edit of the subscription
+            '  `opt_in_ip` varchar(100) DEFAULT NULL,\n' +
+            '  `opt_in_country` varchar(2) DEFAULT NULL,\n' +
+            '  `tz` varchar(100) CHARACTER SET ascii DEFAULT NULL,\n' +
+            '  `status` tinyint(4) unsigned NOT NULL DEFAULT \'1\',\n' +
+            '  `is_test` tinyint(4) unsigned NOT NULL DEFAULT \'0\',\n' +
+            '  `status_change` timestamp NULL DEFAULT NULL,\n' +
+            '  `latest_open` timestamp NULL DEFAULT NULL,\n' +
+            '  `latest_click` timestamp NULL DEFAULT NULL,\n' +
+            '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
+            '  PRIMARY KEY (`id`),\n' +
+            '  UNIQUE KEY `email` (`email`),\n' +
+            '  UNIQUE KEY `cid` (`cid`),\n' +
+            '  KEY `status` (`status`),\n' +
+            '  KEY `subscriber_tz` (`tz`),\n' +
+            '  KEY `is_test` (`is_test`),\n' +
+            '  KEY `latest_open` (`latest_open`),\n' +
+            '  KEY `latest_click` (`latest_click`),\n' +
+            '  KEY `created` (`created`)\n' +
+            ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', entity.id, 'edit');
+
+        const existing = await tx('lists').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'list', 'createList', 'delete');
+
+        await tx('lists').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'list', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', id, 'delete');
+
+        await fields.removeAllByListIdTx(tx, context, id);
+        await segments.removeAllByListIdTx(tx, context, id);
+        await imports.removeAllByListIdTx(tx, context, id);
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            {
+                entityTypeId: 'campaign',
+                query: tx => tx('campaign_lists')
+                    .where('campaign_lists.list', id)
+                    .innerJoin('campaigns', 'campaign_lists.campaign', 'campaigns.id')
+                    .select(['campaigns.id', 'campaigns.name'])
+            }
+        ]);
+
+        await tx('lists').where('id', id).del();
+        await knex.schema.dropTableIfExists('subscription__' + id);
+    });
+}
+
+
+module.exports.UnsubscriptionMode = UnsubscriptionMode;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listWithSegmentByCampaignDTAjax = listWithSegmentByCampaignDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.getByIdWithListFields = getByIdWithListFields;
+module.exports.getByCidTx = getByCidTx;
+module.exports.getByCid = getByCid;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/mosaico-templates.js

@@ -0,0 +1,126 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const files = require('./files');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'type', 'data', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'mosaicoTemplate', id, 'view');
+        const entity = await tx('mosaico_templates').where('id', id).first();
+        entity.data = JSON.parse(entity.data);
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'mosaicoTemplate', id);
+        return entity;
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'mosaicoTemplate', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('mosaico_templates').innerJoin('namespaces', 'namespaces.id', 'mosaico_templates.namespace'),
+        [ 'mosaico_templates.id', 'mosaico_templates.name', 'mosaico_templates.description', 'mosaico_templates.type', 'mosaico_templates.created', 'namespaces.name' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, entity) {
+    entity.data = JSON.stringify(entity.data);
+    await namespaceHelpers.validateEntity(tx, entity);
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createMosaicoTemplate');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const ids = await tx('mosaico_templates').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'mosaicoTemplate', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceGlobalPermission(context, 'createJavascriptWithROAccess');
+        await shares.enforceEntityPermissionTx(tx, context, 'mosaicoTemplate', entity.id, 'edit');
+
+        const existing = await tx('mosaico_templates').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.data = JSON.parse(existing.data);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'mosaicoTemplate', 'createMosaicoTemplate', 'delete');
+
+        await tx('mosaico_templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'mosaicoTemplate', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        const deps = [];
+
+        await shares.enforceEntityPermissionTx(tx, context, 'mosaicoTemplate', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            {
+                entityTypeId: 'template',
+                rows: async (tx, limit) => {
+                    const result = [];
+
+                    const tmpls = await tx('templates').where('type', 'mosaico').select(['id', 'name', 'data']);
+                    for (const tmpl of tmpls) {
+                        const data = JSON.parse(tmpl.data);
+                        if (data.mosaicoTemplate === id) {
+                            result.push(tmpl);
+                        }
+
+                        limit -= 1;
+                        if (limit <= 0) break;
+                    }
+
+                    return result;
+                }
+            }
+        ]);
+
+        await files.removeAllTx(tx, context, 'mosaicoTemplate', 'file', id);
+        await files.removeAllTx(tx, context, 'mosaicoTemplate', 'block', id);
+
+        await tx('mosaico_templates').where('id', id).del();
+    });
+}
+
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/namespaces.js

@@ -0,0 +1,192 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const entitySettings = require('../lib/entity-settings');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+
+const allowedKeys = new Set(['name', 'description', 'namespace']);
+
+async function listTree(context) {
+    // FIXME - process permissions
+
+    const entityType = entitySettings.getEntityType('namespace');
+
+    // This builds a forest of namespaces that contains only those namespace that the user has access to
+    // This goes in three steps: 1) tree with all namespaces is built with parent-children links, 2) the namespaces that are not accessible
+    // by the user are pruned out, which potentially transforms the tree to a forest, 3) unneeded attributes (i.e. parent links)
+    // are removed and children are turned to an array are sorted alphabetically by name
+
+    // Build a tree
+    const rows = await knex('namespaces')
+        .leftJoin(entityType.permissionsTable, {
+            [entityType.permissionsTable + '.entity']: 'namespaces.id',
+            [entityType.permissionsTable + '.user']: context.user.id
+        })
+        .groupBy('namespaces.id')
+        .select([
+            'namespaces.id', 'namespaces.name', 'namespaces.description', 'namespaces.namespace',
+            knex.raw(`GROUP_CONCAT(${entityType.permissionsTable + '.operation'} SEPARATOR \';\') as permissions`)
+        ]);
+
+    const entries = {};
+
+    for (let row of rows) {
+        let entry;
+        if (!entries[row.id]) {
+            entry = {
+                children: {}
+            };
+            entries[row.id] = entry;
+        } else {
+            entry = entries[row.id];
+        }
+
+        if (row.namespace) {
+            if (!entries[row.namespace]) {
+                entries[row.namespace] = {
+                    children: {}
+                };
+            }
+
+            entries[row.namespace].children[row.id] = entry;
+            entry.parent = entries[row.namespace];
+        } else {
+            entry.parent = null;
+        }
+
+        entry.key = row.id;
+        entry.title = row.name;
+        entry.description = row.description;
+        entry.permissions = row.permissions ? row.permissions.split(';') : [];
+    }
+
+    // Prune out the inaccessible namespaces
+    for (const entryId in entries) {
+        const entry = entries[entryId];
+
+        if (!entry.permissions.includes('view')) {
+            for (const childId in entry.children) {
+                const child = entry.children[childId];
+                child.parent = entry.parent;
+
+                if (entry.parent) {
+                    entry.parent.children[childId] = child;
+                }
+            }
+
+            if (entry.parent) {
+                delete entry.parent.children[entryId];
+            }
+
+            delete entries[entryId];
+        }
+    }
+
+    // Retrieve the roots before we discard the parent link
+    const roots = Object.values(entries).filter(x => x.parent === null);
+
+    // Remove parent link, transform children to an array and sort it
+    for (const entryId in entries) {
+        const entry = entries[entryId];
+
+        entry.children = Object.values(entry.children);
+        entry.children.sort((x, y) => x.title.localeCompare(y.title));
+
+        delete entry.parent;
+    }
+
+    return roots;
+}
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', id, 'view');
+        const entity = await tx('namespaces').where('id', id).first();
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'namespace', id);
+        return entity;
+    });
+}
+
+async function create(context, entity) {
+    enforce(entity.namespace, 'Parent namespace must be set');
+
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createNamespace');
+
+        const ids = await tx('namespaces').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        // We don't have to rebuild all entity types, because no entity can be a child of the namespace at this moment.
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'namespace', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    enforce(entity.id !== 1 || entity.namespace === null, 'Cannot assign a parent to the root namespace.');
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.id, 'edit');
+
+        const existing = await tx('namespaces').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        // namespaceHelpers.validateEntity is not needed here because it is part of the tree traversal check below
+        await namespaceHelpers.validateMove(context, entity, existing, 'namespace', 'createNamespace', 'delete');
+
+        let iter = entity;
+        while (iter.namespace != null) {
+            iter = await tx('namespaces').where('id', iter.namespace).first();
+
+            if (!iter) {
+                throw new interoperableErrors.DependencyNotFoundError();
+            }
+
+            if (iter.id === entity.id) {
+                throw new interoperableErrors.LoopDetectedError();
+            }
+        }
+
+        await tx('namespaces').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx);
+    });
+}
+
+async function remove(context, id) {
+    enforce(id !== 1, 'Cannot delete the root namespace.');
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', id, 'delete');
+
+        const entityTypesWithNamespace = Object.keys(entitySettings.getEntityTypes());
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, entityTypesWithNamespace.map(entityTypeId => ({ entityTypeId: entityTypeId, column: 'namespace' })));
+
+        await tx('namespaces').where('id', id).del();
+    });
+}
+
+module.exports.hash = hash;
+module.exports.listTree = listTree;
+module.exports.getById = getById;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/report-templates.js

@@ -0,0 +1,103 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const reports = require('./reports');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', id, 'view');
+        const entity = await tx('report_templates').where('id', id).first();
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'reportTemplate', id);
+        return entity;
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'reportTemplate', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('report_templates').innerJoin('namespaces', 'namespaces.id', 'report_templates.namespace'),
+        [ 'report_templates.id', 'report_templates.name', 'report_templates.description', 'report_templates.created', 'namespaces.name' ]
+    );
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceGlobalPermission(context, 'createJavascriptWithROAccess');
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createReportTemplate');
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        const ids = await tx('report_templates').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'reportTemplate', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceGlobalPermission(context, 'createJavascriptWithROAccess');
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', entity.id, 'edit');
+
+        const existing = await tx('report_templates').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'reportTemplate', 'createReportTemplate', 'delete');
+
+        await tx('report_templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'reportTemplate', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            { entityTypeId: 'report', column: 'report_template' }
+        ]);
+
+        await tx('report_templates').where('id', id).del();
+    });
+}
+
+async function getUserFieldsById(context, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', id, 'view');
+        const entity = await tx('report_templates').select(['user_fields']).where('id', id).first();
+        return JSON.parse(entity.user_fields);
+    });
+}
+
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.getUserFieldsById = getUserFieldsById;

server/models/reports.js

@@ -0,0 +1,208 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const fields = require('./fields');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const reportHelpers = require('../lib/report-helpers');
+const fs = require('fs-extra-promise');
+
+const ReportState = require('../../shared/reports').ReportState;
+
+const allowedKeys = new Set(['name', 'description', 'report_template', 'params', 'namespace']);
+
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getByIdWithTemplate(context, id, withPermissions = true) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'view');
+
+        const entity = await tx('reports')
+            .where('reports.id', id)
+            .innerJoin('report_templates', 'reports.report_template', 'report_templates.id')
+            .select(['reports.id', 'reports.name', 'reports.description', 'reports.report_template', 'reports.params', 'reports.state', 'reports.namespace', 'report_templates.user_fields', 'report_templates.mime_type', 'report_templates.hbs', 'report_templates.js'])
+            .first();
+
+        entity.user_fields = JSON.parse(entity.user_fields);
+        entity.params = JSON.parse(entity.params);
+
+        if (withPermissions) {
+            entity.permissions = await shares.getPermissionsTx(tx, context, 'report', id);
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [
+            { entityTypeId: 'report', requiredOperations: ['view'] },
+            { entityTypeId: 'reportTemplate', requiredOperations: ['view'] }
+        ],
+        params,
+        builder => builder.from('reports')
+            .innerJoin('report_templates', 'reports.report_template', 'report_templates.id')
+            .innerJoin('namespaces', 'namespaces.id', 'reports.namespace'),
+        [
+            'reports.id', 'reports.name', 'report_templates.name', 'reports.description',
+            'reports.last_run', 'namespaces.name', 'reports.state', 'report_templates.mime_type'
+        ]
+    );
+}
+
+async function create(context, entity) {
+    let id;
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createReport');
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', entity.report_template, 'execute');
+
+        await namespaceHelpers.validateEntity(tx, entity);
+
+        entity.params = JSON.stringify(entity.params);
+
+        const ids = await tx('reports').insert(filterObject(entity, allowedKeys));
+        id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'report', entityId: id });
+    });
+
+    const reportProcessor = require('../lib/report-processor');
+    await reportProcessor.start(id);
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'report', entity.id, 'edit');
+        await shares.enforceEntityPermissionTx(tx, context, 'reportTemplate', entity.report_template, 'execute');
+
+        const existing = await tx('reports').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.params = JSON.parse(existing.params);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'report', 'createReport', 'delete');
+
+        entity.params = JSON.stringify(entity.params);
+
+        const filteredUpdates = filterObject(entity, allowedKeys);
+        filteredUpdates.state = ReportState.SCHEDULED;
+
+        await tx('reports').where('id', entity.id).update(filteredUpdates);
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'report', entityId: entity.id });
+    });
+
+    // This require is here to avoid cyclic dependency
+    const reportProcessor = require('../lib/report-processor');
+    await reportProcessor.start(entity.id);
+}
+
+async function removeTx(tx, context, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'report', id, 'delete');
+
+    const report = tx('reports').where('id', id).first();
+
+    await fs.removeAsync(reportHelpers.getReportContentFile(report));
+    await fs.removeAsync(reportHelpers.getReportOutputFile(report));
+
+    await tx('reports').where('id', id).del();
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, id);
+    });
+}
+
+async function updateFields(id, fields) {
+    return await knex('reports').where('id', id).update(fields);
+}
+
+async function listByState(state, limit) {
+    return await knex('reports').where('state', state).limit(limit);
+}
+
+async function bulkChangeState(oldState, newState) {
+    return await knex('reports').where('state', oldState).update('state', newState);
+}
+
+
+const campaignFieldsMapping = {
+    tracker_count: 'campaign_links.count',
+    country: 'campaign_links.country',
+    device_type: 'campaign_links.device_type',
+    status: 'campaign_messages.status',
+    first_name: 'subscriptions.first_name',
+    last_name: 'subscriptions.last_name',
+    email: 'subscriptions.email'
+};
+
+async function getCampaignResults(context, campaign, select, extra) {
+    const flds = await fields.list(context, campaign.list);
+
+    const fieldsMapping = Object.assign({}, campaignFieldsMapping);
+    for (const fld of flds) {
+        /* Dropdown and checkbox groups have field.column == null
+           TODO - For the time being, we don't group options and we don't expand enums. We just provide it as it is in the DB. */
+        if (fld.column) {
+            fieldsMapping[fld.key.toLowerCase()] = 'subscriptions.' + fld.column;
+        }
+    }
+
+    let selFields = [];
+    for (let idx = 0; idx < select.length; idx++) {
+        const item = select[idx];
+        if (item in fieldsMapping) {
+            selFields.push(fieldsMapping[item] + ' AS ' + item);
+        } else if (item === '*') {
+            selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
+        } else {
+            selFields.push(item);
+        }
+    }
+
+    let query = knex(`subscription__${campaign.list} AS subscriptions`)
+        .innerJoin('campaign_messages', 'subscriptions.id', 'campaign_messages.subscription')
+        .leftJoin('campaign_links', 'subscriptions.id', 'campaign_links.subscription')
+        .where('campaign_messages.list', campaign.list)
+        .where('campaign_links.list', campaign.list)
+        .select(selFields);
+
+    if (extra) {
+        query = extra(query);
+    }
+
+    return await query;
+}
+
+
+
+module.exports.ReportState = ReportState;
+module.exports.hash = hash;
+module.exports.getByIdWithTemplate = getByIdWithTemplate;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.updateFields = updateFields;
+module.exports.listByState = listByState;
+module.exports.bulkChangeState = bulkChangeState;
+module.exports.getCampaignResults = getCampaignResults;

server/models/segments.js

@@ -0,0 +1,433 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const { enforce, filterObject } = require('../lib/helpers');
+const hasher = require('node-object-hash')();
+const moment = require('moment');
+const fields = require('./fields');
+const subscriptions = require('./subscriptions');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const { parseDate, parseBirthday, DateFormat } = require('../../shared/date');
+
+const allowedKeys = new Set(['name', 'settings']);
+
+
+
+const predefColumns = [
+    {
+        column: 'email',
+        type: 'text'
+    },
+    {
+        column: 'opt_in_country',
+        type: 'text'
+    },
+    {
+        column: 'created',
+        type: 'date'
+    },
+    {
+        column: 'latest_open',
+        type: 'date'
+    },
+    {
+        column: 'latest_click',
+        type: 'date'
+    }
+];
+
+
+const compositeRuleTypes = {
+    all: {
+        addQuery: (query, rules, addSubQuery) => {
+            for (const rule of rules) {
+                query.where(function() {
+                    addSubQuery(this, rule);
+                });
+            }
+        }
+    },
+    some: {
+        addQuery: (query, rules, addSubQuery) => {
+            for (const rule of rules) {
+                query.orWhere(function() {
+                    addSubQuery(this, rule);
+                });
+            }
+        }
+    },
+    none: {
+        addQuery: (query, rules, addSubQuery) => {
+            for (const rule of rules) {
+                query.whereNot(function() {
+                    addSubQuery(this, rule);
+                });
+            }
+        }
+    },
+};
+
+
+
+const primitiveRuleTypes = {
+    text: {},
+    website: {},
+    number: {},
+    date: {},
+    birthday: {},
+    option: {},
+    'dropdown-enum': {},
+    'radio-enum': {}
+};
+
+
+function stringValueSettings(sqlOperator, allowEmpty) {
+    return {
+        validate: rule => {
+            enforce(typeof rule.value === 'string', 'Invalid value type in rule');
+            enforce(allowEmpty || rule.value, 'Value in rule must not be empty');
+        },
+        addQuery: (subsTableName, query, rule) => query.where(subsTableName + '. ' + rule.column, sqlOperator, rule.value)
+    };
+}
+
+function numberValueSettings(sqlOperator) {
+    return {
+        validate: rule => {
+            enforce(typeof rule.value === 'number', 'Invalid value type in rule');
+        },
+        addQuery: (subsTableName, query, rule) => query.where(subsTableName + '. ' + rule.column, sqlOperator, rule.value)
+    };
+}
+
+function dateValueSettings(thisDaySqlOperator, nextDaySqlOperator) {
+    return {
+        validate: rule => {
+            const date = moment.utc(rule.value);
+            enforce(date.isValid(), 'Invalid date value');
+        },
+        addQuery: (subsTableName, query, rule) => {
+            const thisDay = moment.utc(rule.value).startOf('day');
+            const nextDay = moment(thisDay).add(1, 'days');
+
+            if (thisDaySqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, thisDaySqlOperator, thisDay.toDate())
+            }
+
+            if (nextDaySqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, nextDaySqlOperator, nextDay.toDate());
+            }
+        }
+    };
+}
+
+function dateRelativeValueSettings(todaySqlOperator, tomorrowSqlOperator) {
+    return {
+        validate: rule => {
+            enforce(typeof rule.value === 'number', 'Invalid value type in rule');
+        },
+        addQuery: (subsTableName, query, rule) => {
+            const todayWithOffset = moment.utc().startOf('day').add(rule.value, 'days');
+            const tomorrowWithOffset = moment(todayWithOffset).add(1, 'days');
+
+            if (todaySqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, todaySqlOperator, todayWithOffset.toDate())
+            }
+
+            if (tomorrowSqlOperator) {
+                query.where(subsTableName + '. ' + rule.column, tomorrowSqlOperator, tomorrowWithOffset.toDate());
+            }
+        }
+    };
+}
+
+function optionValueSettings(value) {
+    return {
+        validate: rule => {},
+        addQuery: (subsTableName, query, rule) => query.where(subsTableName + '. ' + rule.column, value)
+    };
+}
+
+
+primitiveRuleTypes.text.eq = stringValueSettings('=', true);
+primitiveRuleTypes.text.like = stringValueSettings('LIKE', true);
+primitiveRuleTypes.text.re = stringValueSettings('REGEXP', true);
+primitiveRuleTypes.text.lt = stringValueSettings('<', false);
+primitiveRuleTypes.text.le = stringValueSettings('<=', false);
+primitiveRuleTypes.text.gt = stringValueSettings('>', false);
+primitiveRuleTypes.text.ge = stringValueSettings('>=', false);
+
+primitiveRuleTypes.website.eq = stringValueSettings('=', true);
+primitiveRuleTypes.website.like = stringValueSettings('LIKE', true);
+primitiveRuleTypes.website.re = stringValueSettings('REGEXP', true);
+
+primitiveRuleTypes.number.eq = numberValueSettings('=');
+primitiveRuleTypes.number.lt = numberValueSettings('<');
+primitiveRuleTypes.number.le = numberValueSettings('<=');
+primitiveRuleTypes.number.gt = numberValueSettings('>');
+primitiveRuleTypes.number.ge = numberValueSettings('>=');
+
+primitiveRuleTypes.date.eq = dateValueSettings('>=', '<');
+primitiveRuleTypes.date.lt = dateValueSettings('<', null);
+primitiveRuleTypes.date.le = dateValueSettings(null, '<');
+primitiveRuleTypes.date.gt = dateValueSettings(null, '>=');
+primitiveRuleTypes.date.ge = dateValueSettings('>=', null);
+
+primitiveRuleTypes.date.eqTodayPlusDays = dateRelativeValueSettings('>=', '<');
+primitiveRuleTypes.date.ltTodayPlusDays = dateRelativeValueSettings('<', null);
+primitiveRuleTypes.date.leTodayPlusDays = dateRelativeValueSettings(null, '<');
+primitiveRuleTypes.date.gtTodayPlusDays = dateRelativeValueSettings(null, '>=');
+primitiveRuleTypes.date.geTodayPlusDays = dateRelativeValueSettings('>=', null);
+
+primitiveRuleTypes.birthday.eq = dateValueSettings('>=', '<');
+primitiveRuleTypes.birthday.lt = dateValueSettings('<', null);
+primitiveRuleTypes.birthday.le = dateValueSettings(null, '<');
+primitiveRuleTypes.birthday.gt = dateValueSettings(null, '>=');
+primitiveRuleTypes.birthday.ge = dateValueSettings('>=', null);
+
+primitiveRuleTypes.option.isTrue = optionValueSettings(true);
+primitiveRuleTypes.option.isFalse = optionValueSettings(false);
+
+primitiveRuleTypes['dropdown-enum'].eq = stringValueSettings('=', true);
+primitiveRuleTypes['dropdown-enum'].like = stringValueSettings('LIKE', true);
+primitiveRuleTypes['dropdown-enum'].re = stringValueSettings('REGEXP', true);
+primitiveRuleTypes['dropdown-enum'].lt = stringValueSettings('<', false);
+primitiveRuleTypes['dropdown-enum'].le = stringValueSettings('<=', false);
+primitiveRuleTypes['dropdown-enum'].gt = stringValueSettings('>', false);
+primitiveRuleTypes['dropdown-enum'].ge = stringValueSettings('>=', false);
+
+primitiveRuleTypes['radio-enum'].eq = stringValueSettings('=', true);
+primitiveRuleTypes['radio-enum'].like = stringValueSettings('LIKE', true);
+primitiveRuleTypes['radio-enum'].re = stringValueSettings('REGEXP', true);
+primitiveRuleTypes['radio-enum'].lt = stringValueSettings('<', false);
+primitiveRuleTypes['radio-enum'].le = stringValueSettings('<=', false);
+primitiveRuleTypes['radio-enum'].gt = stringValueSettings('>', false);
+primitiveRuleTypes['radio-enum'].ge = stringValueSettings('>=', false);
+
+
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function listDTAjax(context, listId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSegments');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('segments')
+                .where('list', listId),
+            ['id', 'name']
+        );
+    });
+}
+
+async function listIdName(context, listId) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSegments']);
+
+        return await tx('segments').select(['id', 'name']).where('list', listId).orderBy('name', 'asc');
+    });
+}
+
+async function getByIdTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSegments');
+    const entity = await tx('segments').where({id, list: listId}).first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    entity.settings = JSON.parse(entity.settings);
+    return entity;
+}
+
+async function getById(context, listId, id) {
+    return await knex.transaction(async tx => {
+        return getByIdTx(tx, context, listId, id);
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, entity, isCreate) {
+    enforce(entity.name, 'Name must be present');
+    enforce(entity.settings, 'Settings must be present');
+    enforce(entity.settings.rootRule, 'Root rule must be present in setting');
+    enforce(entity.settings.rootRule.type in compositeRuleTypes, 'Root rule must be composite');
+
+
+    const flds = await fields.listTx(tx, listId);
+    const allowedFlds = [
+        ...predefColumns,
+        ...flds.filter(fld => fld.type in primitiveRuleTypes)
+    ];
+
+    const fieldsByColumn = {};
+    for (const fld of allowedFlds) {
+        fieldsByColumn[fld.column] = fld;
+    }
+
+    function validateRule(rule) {
+        if (rule.type in compositeRuleTypes) {
+            for (const childRule of rule.rules) {
+                validateRule(childRule);
+            }
+        } else {
+            const colType = fieldsByColumn[rule.column].type;
+            primitiveRuleTypes[colType][rule.type].validate(rule);
+        }
+    }
+
+
+    validateRule(entity.settings.rootRule);
+
+    entity.settings = JSON.stringify(entity.settings);
+}
+
+async function create(context, listId, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+        await _validateAndPreprocess(tx, listId, entity, true);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.list = listId;
+
+        const ids = await tx('segments').insert(filteredEntity);
+        const id = ids[0];
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, listId, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+        const existing = await tx('segments').where({list: listId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.settings = JSON.parse(existing.settings);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, listId, entity, false);
+
+        await tx('segments').where({list: listId, id: entity.id}).update(filterObject(entity, allowedKeys));
+    });
+}
+
+
+async function removeTx(tx, context, listId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+    await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+        {
+            entityTypeId: 'campaign',
+            query: tx => tx('campaign_lists')
+                .where('campaign_lists.segment', id)
+                .innerJoin('campaigns', 'campaign_lists.campaign', 'campaigns.id')
+                .select(['campaigns.id', 'campaigns.name'])
+        }
+    ]);
+
+    // The listId "where" is here to prevent deleting segment of a list for which a user does not have permission
+    await tx('segments').where({list: listId, id}).del();
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, listId, id);
+    });
+}
+
+async function removeAllByListIdTx(tx, context, listId) {
+    const entities = await tx('segments').where('list', listId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, listId, entity.id);
+    }
+}
+
+async function removeRulesByColumnTx(tx, context, listId, column) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSegments');
+
+    function pruneChildRules(rule) {
+        if (rule.type in compositeRuleTypes) {
+
+            const newRules = [];
+
+            for (const childRule of rule.rules) {
+                if (childRule.column !== column) {
+                    pruneChildRules(childRule);
+                    newRules.push(childRule);
+                }
+            }
+
+            rule.rules = newRules;
+        }
+    }
+
+    const entities = await tx('segments').where({list: listId});
+    for (const entity of entities) {
+        const settings = JSON.parse(entity.settings);
+
+        pruneChildRules(settings.rootRule);
+
+        await tx('segments').where({list: listId, id: entity.id}).update('settings', JSON.stringify(settings));
+    }
+}
+
+async function getQueryGeneratorTx(tx, listId, id) {
+    const flds = await fields.listTx(tx, listId);
+    const allowedFlds = [
+        ...predefColumns,
+        ...flds.filter(fld => fld.type in primitiveRuleTypes)
+    ];
+
+    const fieldsByColumn = {};
+    for (const fld of allowedFlds) {
+        fieldsByColumn[fld.column] = fld;
+    }
+
+    const entity = await tx('segments').where({id, list: listId}).first();
+    const settings = JSON.parse(entity.settings);
+
+    const subsTableName = subscriptions.getSubscriptionTableName(listId);
+
+    function processRule(query, rule) {
+        if (rule.type in compositeRuleTypes) {
+            compositeRuleTypes[rule.type].addQuery(query, rule.rules, (subQuery, childRule) => {
+                processRule(subQuery, childRule);
+            });
+        } else {
+            const colType = fieldsByColumn[rule.column].type;
+            primitiveRuleTypes[colType][rule.type].addQuery(subsTableName, query, rule);
+        }
+    }
+
+    return query => processRule(query, settings.rootRule);
+} 
+
+// This is to handle circular dependency with fields.js
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listIdName = listIdName;
+module.exports.getById = getById;
+module.exports.getByIdTx = getByIdTx;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeAllByListIdTx = removeAllByListIdTx;
+module.exports.removeRulesByColumnTx = removeRulesByColumnTx;
+module.exports.getQueryGeneratorTx = getQueryGeneratorTx;

server/models/send-configurations.js

@@ -0,0 +1,186 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const dtHelpers = require('../lib/dt-helpers');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const {MailerType, getSystemSendConfigurationId} = require('../../shared/send-configurations');
+const contextHelpers = require('../lib/context-helpers');
+const mailers = require('../lib/mailers');
+const senders = require('../lib/senders');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable', 'x_mailer', 'verp_hostname', 'mailer_type', 'mailer_settings', 'namespace']);
+
+const allowedMailerTypes = new Set(Object.values(MailerType));
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'sendConfiguration', requiredOperations: ['viewPublic'] }],
+        params,
+        builder => builder
+            .from('send_configurations')
+            .innerJoin('namespaces', 'namespaces.id', 'send_configurations.namespace'),
+        ['send_configurations.id', 'send_configurations.name', 'send_configurations.cid', 'send_configurations.description', 'send_configurations.mailer_type', 'send_configurations.created', 'namespaces.name']
+    );
+}
+
+async function listWithSendPermissionDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'sendConfiguration', requiredOperations: ['sendWithoutOverrides', 'sendWithAllowedOverrides', 'sendWithAnyOverrides'] }],
+        params,
+        builder => builder
+            .from('send_configurations')
+            .innerJoin('namespaces', 'namespaces.id', 'send_configurations.namespace'),
+        ['send_configurations.id', 'send_configurations.name', 'send_configurations.cid', 'send_configurations.description', 'send_configurations.mailer_type', 'send_configurations.created', 'namespaces.name']
+    );
+}
+
+async function _getByTx(tx, context, key, id, withPermissions, withPrivateData) {
+    let entity;
+
+    if (withPrivateData) {
+        entity = await tx('send_configurations').where(key, id).first();
+
+        if (!entity) {
+            shares.throwPermissionDenied();
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', entity.id, 'viewPrivate');
+
+        entity.mailer_settings = JSON.parse(entity.mailer_settings);
+    } else {
+        entity = await tx('send_configurations').where(key, id).select(
+            ['id', 'name', 'cid', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'reply_to', 'reply_to_overridable', 'subject', 'subject_overridable']
+        ).first();
+
+        if (!entity) {
+            shares.throwPermissionDenied();
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', entity.id, 'viewPublic');
+    }
+
+    // note that permissions are optional as as this methods may be used with synthetic admin context
+    if (withPermissions) {
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'sendConfiguration', id);
+    }
+
+    return entity;
+
+}
+
+async function getByIdTx(tx, context, id, withPermissions = true, withPrivateData = true) {
+    return await _getByTx(tx, context, 'id', id, withPermissions, withPrivateData);
+}
+
+async function getById(context, id, withPermissions = true, withPrivateData = true) {
+    return await knex.transaction(async tx => {
+        return await getByIdTx(tx, context, id, withPermissions, withPrivateData);
+    });
+}
+
+async function getByCid(context, cid, withPermissions = true, withPrivateData = true) {
+    return await knex.transaction(async tx => {
+        return await _getByTx(tx, context, 'cid', cid, withPermissions, withPrivateData);
+    });
+}
+
+async function _validateAndPreprocess(tx, entity, isCreate) {
+    await namespaceHelpers.validateEntity(tx, entity);
+
+    enforce(allowedMailerTypes.has(entity.mailer_type), 'Unknown mailer type');
+    entity.mailer_settings = JSON.stringify(entity.mailer_settings);
+}
+
+
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createSendConfiguration');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.cid = shortid.generate();
+
+        const ids = await tx('send_configurations').insert(filteredEntity);
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'sendConfiguration', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', entity.id, 'edit');
+
+        const existing = await tx('send_configurations').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.mailer_settings = JSON.parse(existing.mailer_settings);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'sendConfiguration', 'createSendConfiguration', 'delete');
+
+        await tx('send_configurations').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'sendConfiguration', entityId: entity.id });
+    });
+
+    mailers.invalidateMailer(entity.id);
+    senders.reloadConfig(entity.id);
+}
+
+async function remove(context, id) {
+    if (id === getSystemSendConfigurationId()) {
+        shares.throwPermissionDenied();
+    }
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            { entityTypeId: 'campaign', column: 'send_configuration' },
+            { entityTypeId: 'list', column: 'send_configuration' }
+        ]);
+
+        await tx('send_configurations').where('id', id).del();
+    });
+}
+
+async function getSystemSendConfiguration() {
+    return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId(), false);
+}
+
+module.exports.MailerType = MailerType;
+module.exports.hash = hash;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listWithSendPermissionDTAjax = listWithSendPermissionDTAjax;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.getByCid = getByCid;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.getSystemSendConfiguration = getSystemSendConfiguration;

server/models/settings.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { filterObject } = require('../lib/helpers');
+const hasher = require('node-object-hash')();
+const shares = require('./shares');
+
+const allowedKeys = new Set(['adminEmail', 'uaCode', 'shoutout', 'pgpPassphrase', 'pgpPrivateKey', 'defaultHomepage']);
+// defaultHomepage is used as a default to list.homepage - if the list.homepage is not filled in
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function get(context, keyOrKeys) {
+    shares.enforceGlobalPermission(context, 'manageSettings');
+
+    let keys;
+    if (!keyOrKeys) {
+        keys = [...allowedKeys.values()];
+    } else if (!Array.isArray(keyOrKeys)) {
+        keys = [ keys ];
+    } else {
+        keys = keyOrKeys;
+    }
+
+    const rows = await knex('settings').select(['key', 'value']).whereIn('key', keys);
+
+    const settings = {};
+    for (const row of rows) {
+        settings[row.key] = row.value;
+    }
+
+    if (!Array.isArray(keyOrKeys) && keyOrKeys) {
+        return settings[keyOrKeys];
+    } else {
+        return settings;
+    }
+}
+
+async function set(context, data) {
+    shares.enforceGlobalPermission(context, 'manageSettings');
+
+    for (const key in data) {
+        if (allowedKeys.has(key)) {
+            const value = data[key];
+            try {
+                await knex('settings').insert({key, value});
+            } catch (err) {
+                await knex('settings').where('key', key).update('value', value);
+            }
+        }
+    }
+
+    // FIXME - recreate mailers, notify senders to recreate the mailers
+}
+
+module.exports.hash = hash;
+module.exports.get = get;
+module.exports.set = set;
+

server/models/shares.js

@@ -0,0 +1,712 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const config = require('config');
+const { enforce } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const entitySettings = require('../lib/entity-settings');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const log = require('../lib/log');
+const {getGlobalNamespaceId} = require('../../shared/namespaces');
+const {getAdminId} = require('../../shared/users');
+
+// TODO: This would really benefit from some permission cache connected to rebuildPermissions
+// A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions
+
+
+async function listByEntityDTAjax(context, entityTypeId, entityId, params) {
+    return await knex.transaction(async (tx) => {
+        const entityType = entitySettings.getEntityType(entityTypeId);
+        await enforceEntityPermissionTx(tx, context, entityTypeId, entityId, 'share');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from(entityType.sharesTable)
+                .innerJoin('users', entityType.sharesTable + '.user', 'users.id')
+                .innerJoin('generated_role_names', 'generated_role_names.role', 'users.role')
+                .where('generated_role_names.entity_type', entityTypeId)
+                .where(`${entityType.sharesTable}.entity`, entityId),
+            ['users.username', 'users.name', 'generated_role_names.name', 'users.id', entityType.sharesTable + '.auto']
+        );
+    });
+}
+
+async function listByUserDTAjax(context, entityTypeId, userId, params) {
+    return await knex.transaction(async (tx) => {
+        const user = await tx('users').where('id', userId).first();
+        if (!user) {
+            shares.throwPermissionDenied();
+        }
+
+        await enforceEntityPermissionTx(tx, context, 'namespace', user.namespace, 'manageUsers');
+
+        const entityType = entitySettings.getEntityType(entityTypeId);
+
+        return await dtHelpers.ajaxListWithPermissionsTx(
+            tx,
+            context,
+            [{entityTypeId}],
+            params,
+            builder => builder
+                .from(entityType.sharesTable)
+                .innerJoin(entityType.entitiesTable, entityType.sharesTable + '.entity', entityType.entitiesTable + '.id')
+                .innerJoin('generated_role_names', 'generated_role_names.role', entityType.sharesTable + '.role')
+                .where('generated_role_names.entity_type', entityTypeId)
+                .where(entityType.sharesTable + '.user', userId),
+            [entityType.entitiesTable + '.name', 'generated_role_names.name', entityType.entitiesTable + '.id', entityType.sharesTable + '.auto']
+        );
+    });
+}
+
+async function listUnassignedUsersDTAjax(context, entityTypeId, entityId, params) {
+    return await knex.transaction(async (tx) => {
+        const entityType = entitySettings.getEntityType(entityTypeId);
+
+        await enforceEntityPermissionTx(tx, context, entityTypeId, entityId, 'share');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('users')
+                .whereNotExists(function () {
+                    return this
+                        .select('*')
+                        .from(entityType.sharesTable)
+                        .whereRaw(`users.id = ${entityType.sharesTable}.user`)
+                        .andWhere(`${entityType.sharesTable}.entity`, entityId);
+                }),
+            ['users.id', 'users.username', 'users.name']
+        );
+    });
+}
+
+async function listRolesDTAjax(entityTypeId, params) {
+    return await dtHelpers.ajaxList(
+        params,
+        builder => builder
+            .from('generated_role_names')
+            .where({entity_type: entityTypeId}),
+        ['role', 'name', 'description']
+    );
+}
+
+async function assign(context, entityTypeId, entityId, userId, role) {
+    const entityType = entitySettings.getEntityType(entityTypeId);
+
+    await knex.transaction(async tx => {
+        await enforceEntityPermissionTx(tx, context, entityTypeId, entityId, 'share');
+
+        enforce(await tx('users').where('id', userId).select('id').first(), 'Invalid user id');
+
+        const extraColumns = entityType.dependentPermissions ? entityType.dependentPermissions.extraColumns : [];
+        const entity = await tx(entityType.entitiesTable).where('id', entityId).select(['id', ...extraColumns]).first();
+        enforce(entity, 'Invalid entity id');
+
+        if (entityType.dependentPermissions) {
+            enforce(!entityType.dependentPermissions.getParent(entity), 'Cannot share/unshare a dependent entity');
+        }
+
+        const entry = await tx(entityType.sharesTable).where({user: userId, entity: entityId}).select('role').first();
+
+        if (entry) {
+            if (!role) {
+                await tx(entityType.sharesTable).where({user: userId, entity: entityId}).del();
+            } else if (entry.role !== role) {
+                await tx(entityType.sharesTable).where({user: userId, entity: entityId}).update('role', role);
+            }
+        } else {
+            await tx(entityType.sharesTable).insert({
+                user: userId,
+                entity: entityId,
+                role
+            });
+        }
+
+        await tx(entityType.permissionsTable).where({user: userId, entity: entityId}).del();
+        if (entityTypeId === 'namespace') {
+            await rebuildPermissionsTx(tx, {userId});
+        } else if (role) {
+            await rebuildPermissionsTx(tx, { entityTypeId, entityId, userId });
+        }
+    });
+}
+
+async function rebuildPermissionsTx(tx, restriction) {
+    restriction = restriction || {};
+
+    const namespaceEntityType = entitySettings.getEntityType('namespace');
+
+    // Collect entity types we care about
+    let restrictedEntityTypes;
+    if (restriction.entityTypeId) {
+        const entityType = entitySettings.getEntityType(restriction.entityTypeId);
+        restrictedEntityTypes = {
+            [restriction.entityTypeId]: entityType
+        };
+    } else {
+        restrictedEntityTypes = entitySettings.getEntityTypesWithPermissions();
+    }
+
+
+    // To prevent users locking out themselves, we consider user with id 1 to be the admin and always assign it
+    // the admin role. The admin role is a global role that has admin===true
+    // If this behavior is not desired, it is enough to delete the user with id 1.
+    const adminUser = await tx('users').where('id', getAdminId()).first();
+    if (adminUser) {
+        let adminRole;
+        for (const role in config.roles.global) {
+            if (config.roles.global[role].admin) {
+                adminRole = role;
+                break;
+            }
+        }
+
+        if (adminRole) {
+            await tx('users').update('role', adminRole).where('id', getAdminId());
+        }
+    }
+
+
+    // Reset root and own namespace shares as per the user roles
+    const usersWithRoleInOwnNamespaceQuery = tx('users')
+        .leftJoin(namespaceEntityType.sharesTable, {
+            'users.id': `${namespaceEntityType.sharesTable}.user`,
+            'users.namespace': `${namespaceEntityType.sharesTable}.entity`
+        })
+        .select(['users.id', 'users.namespace', 'users.role as userRole', `${namespaceEntityType.sharesTable}.role`]);
+    if (restriction.userId) {
+        usersWithRoleInOwnNamespaceQuery.where('users.id', restriction.userId);
+    }
+    const usersWithRoleInOwnNamespace = await usersWithRoleInOwnNamespaceQuery;
+
+    for (const user of usersWithRoleInOwnNamespace) {
+        const roleConf = config.roles.global[user.userRole];
+
+        if (roleConf) {
+            const desiredRole = roleConf.ownNamespaceRole;
+            if (desiredRole && user.role !== desiredRole) {
+                await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: user.namespace }).del();
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: user.namespace, role: desiredRole, auto: true });
+            }
+        }
+    }
+
+
+    const usersWithRoleInRootNamespaceQuery = tx('users')
+        .leftJoin(namespaceEntityType.sharesTable, {
+            'users.id': `${namespaceEntityType.sharesTable}.user`,
+            [`${namespaceEntityType.sharesTable}.entity`]: getGlobalNamespaceId()
+        })
+        .select(['users.id', 'users.role as userRole', `${namespaceEntityType.sharesTable}.role`]);
+    if (restriction.userId) {
+        usersWithRoleInRootNamespaceQuery.andWhere('users.id', restriction.userId);
+    }
+    const usersWithRoleInRootNamespace = await usersWithRoleInRootNamespaceQuery;
+
+    for (const user of usersWithRoleInRootNamespace) {
+        const roleConf = config.roles.global[user.userRole];
+
+        if (roleConf) {
+            const desiredRole = roleConf.rootNamespaceRole;
+            if (desiredRole && user.role !== desiredRole) {
+                await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: getGlobalNamespaceId() }).del();
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: getGlobalNamespaceId(), role: desiredRole, auto: 1 });
+            }
+        }
+    }
+
+
+
+    // Build the map of all namespaces
+    // nsMap is a map of namespaces - each of the following shape:
+    // .id - id of the namespace
+    // .namespace - id of the parent or null if no parent
+    // .userPermissions - Map userId -> [entityTypeId] -> array of permissions
+    // .transitiveUserPermissions - the same as above, but taking into account transitive permission obtained from namespace parents
+
+    const namespaces = await tx('namespaces').select(['id', 'namespace']);
+
+    const nsMap = new Map();
+    for (const namespace of namespaces) {
+        namespace.userPermissions = new Map();
+        nsMap.set(namespace.id, namespace);
+    }
+
+    // This populates .userPermissions
+    const nsSharesQuery = tx(namespaceEntityType.sharesTable).select(['entity', 'user', 'role']);
+    if (restriction.userId) {
+        nsSharesQuery.where('user', restriction.userId);
+    }
+
+    const nsShares = await nsSharesQuery;
+    for (const nsShare of nsShares) {
+        const ns = nsMap.get(nsShare.entity);
+
+        const userPerms = {};
+        ns.userPermissions.set(nsShare.user, userPerms);
+
+        for (const entityTypeId in restrictedEntityTypes) {
+            if (config.roles.namespace[nsShare.role] &&
+                config.roles.namespace[nsShare.role].children &&
+                config.roles.namespace[nsShare.role].children[entityTypeId]) {
+
+                userPerms[entityTypeId] = new Set(config.roles.namespace[nsShare.role].children[entityTypeId]);
+
+            } else {
+                userPerms[entityTypeId] = new Set();
+            }
+        }
+    }
+
+    // This computes .transitiveUserPermissions
+    for (const ns of nsMap.values()) {
+        ns.transitiveUserPermissions = new Map();
+
+        for (const userPermsPair of ns.userPermissions) {
+            const userPerms = {};
+            ns.transitiveUserPermissions.set(userPermsPair[0], userPerms);
+
+            for (const entityTypeId in restrictedEntityTypes) {
+                userPerms[entityTypeId] = new Set(userPermsPair[1][entityTypeId]);
+            }
+        }
+
+        let parentId = ns.namespace;
+        while (parentId) {
+            const parent = nsMap.get(parentId);
+
+            for (const userPermsPair of parent.userPermissions) {
+                const user = userPermsPair[0];
+
+                if (ns.transitiveUserPermissions.has(user)) {
+                    const userPerms = ns.transitiveUserPermissions.get(user);
+
+                    for (const entityTypeId in restrictedEntityTypes) {
+                        for (const perm of userPermsPair[1][entityTypeId]) {
+                            userPerms[entityTypeId].add(perm);
+                        }
+                    }
+                } else {
+                    const userPerms = {};
+                    ns.transitiveUserPermissions.set(user, userPerms);
+
+                    for (const entityTypeId in restrictedEntityTypes) {
+                        userPerms[entityTypeId] = new Set(userPermsPair[1][entityTypeId]);
+                    }
+                }
+            }
+
+            parentId = parent.namespace;
+        }
+    }
+
+
+    // This reads direct shares from DB, joins each with the permissions from namespaces and stores the permissions into DB
+    for (const entityTypeId in restrictedEntityTypes) {
+        const entityType = restrictedEntityTypes[entityTypeId];
+
+        const expungeQuery = tx(entityType.permissionsTable).del();
+        if (restriction.entityId) {
+            expungeQuery.where('entity', restriction.entityId);
+        }
+        if (restriction.userId) {
+            expungeQuery.where('user', restriction.userId);
+        }
+        await expungeQuery;
+
+        const extraColumns = entityType.dependentPermissions ? entityType.dependentPermissions.extraColumns : [];
+        const entitiesQuery = tx(entityType.entitiesTable).select(['id', 'namespace', ...extraColumns]);
+
+
+        const notToBeInserted = new Set();
+        if (restriction.entityId) {
+            if (restriction.parentId) {
+                notToBeInserted.add(restriction.parentId);
+                entitiesQuery.whereIn('id', [restriction.entityId, restriction.parentId]);
+            } else {
+                entitiesQuery.where('id', restriction.entityId);
+            }
+        }
+        const entities = await entitiesQuery;
+
+        // TODO - process restriction.parentId
+
+        const parentEntities = new Map();
+        let nonChildEntities;
+        if (entityType.dependentPermissions) {
+            nonChildEntities = [];
+
+            for (const entity of entities) {
+                const parent = entityType.dependentPermissions.getParent(entity);
+
+                if (parent) {
+                    let childEntities;
+                    if (parentEntities.has(parent)) {
+                        childEntities = parentEntities.get(parent);
+                    } else {
+                        childEntities = [];
+                        parentEntities.set(parent, childEntities);
+                    }
+
+                    childEntities.push(entity.id);
+                } else {
+                    nonChildEntities.push(entity);
+                }
+            }
+        } else {
+            nonChildEntities = entities;
+        }
+
+
+        for (const entity of nonChildEntities) {
+            const permsPerUser = new Map();
+
+            if (entity.namespace) { // The root namespace has not parent namespace, thus the test
+                const transitiveUserPermissions = nsMap.get(entity.namespace).transitiveUserPermissions;
+                for (const transitivePermsPair of transitiveUserPermissions.entries()) {
+                    permsPerUser.set(transitivePermsPair[0], new Set(transitivePermsPair[1][entityTypeId]));
+                }
+            }
+
+            const directSharesQuery = tx(entityType.sharesTable).select(['user', 'role']).where('entity', entity.id);
+            if (restriction.userId) {
+                directSharesQuery.andWhere('user', restriction.userId);
+            }
+            const directShares = await directSharesQuery;
+
+            for (const share of directShares) {
+                let userPerms;
+                if (permsPerUser.has(share.user)) {
+                    userPerms = permsPerUser.get(share.user);
+                } else {
+                    userPerms = new Set();
+                    permsPerUser.set(share.user, userPerms);
+                }
+
+                if (config.roles[entityTypeId][share.role] &&
+                    config.roles[entityTypeId][share.role].permissions) {
+
+                    for (const perm of config.roles[entityTypeId][share.role].permissions) {
+                        userPerms.add(perm);
+                    }
+                }
+            }
+
+            if (!notToBeInserted.has(entity.id)) {
+                for (const userPermsPair of permsPerUser.entries()) {
+                    const data = [];
+
+                    for (const operation of userPermsPair[1]) {
+                        data.push({user: userPermsPair[0], entity: entity.id, operation});
+                    }
+
+                    if (data.length > 0) {
+                        await tx(entityType.permissionsTable).insert(data);
+                    }
+                }
+            }
+
+            if (parentEntities.has(entity.id)) {
+                const childEntities = parentEntities.get(entity.id);
+
+                for (const childId of childEntities) {
+                    for (const userPermsPair of permsPerUser.entries()) {
+                        const data = [];
+
+                        for (const operation of userPermsPair[1]) {
+                            if (operation !== 'share') {
+                                data.push({user: userPermsPair[0], entity: childId, operation});
+                            }
+                        }
+
+                        if (data.length > 0) {
+                            await tx(entityType.permissionsTable).insert(data);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+async function rebuildPermissions(restriction) {
+    await knex.transaction(async tx => {
+        await rebuildPermissionsTx(tx, restriction);
+    });
+}
+
+async function regenerateRoleNamesTable() {
+    await knex.transaction(async tx => {
+        await tx('generated_role_names').del();
+
+        const entityTypeIds = ['global', ...Object.keys(entitySettings.getEntityTypesWithPermissions())];
+
+        for (const entityTypeId of entityTypeIds) {
+            const roles = config.roles[entityTypeId];
+
+            for (const role in roles) {
+                await tx('generated_role_names').insert({
+                    entity_type: entityTypeId,
+                    role,
+                    name: roles[role].name,
+                    description: roles[role].description,
+                });
+            }
+        }
+    });
+}
+
+
+function throwPermissionDenied() {
+    throw new interoperableErrors.PermissionDeniedError('Permission denied');
+}
+
+async function removeDefaultShares(tx, user) {
+    const namespaceEntityType = entitySettings.getEntityType('namespace');
+
+    const roleConf = config.roles.global[user.role];
+
+    if (roleConf) {
+        const desiredRole = roleConf.rootNamespaceRole;
+
+        if (roleConf.ownNamespaceRole) {
+            await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: user.namespace }).del();
+        }
+
+        if (roleConf.rootNamespaceRole) {
+            await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: getGlobalNamespaceId() }).del();
+        }
+    }
+}
+
+function checkGlobalPermission(context, requiredOperations) {
+    if (!context.user) {
+        return false;
+    }
+
+    if (typeof requiredOperations === 'string') {
+        requiredOperations = [ requiredOperations ];
+    }
+
+    if (context.user.restrictedAccessHandler) {
+        const originalRequiredOperations = requiredOperations;
+        const allowedPerms = context.user.restrictedAccessHandler.globalPermissions;
+        if (allowedPerms) {
+            requiredOperations = requiredOperations.filter(perm => allowedPerms.has(perm));
+        } else {
+            requiredOperations = [];
+        }
+        log.verbose('check global permissions with restrictedAccessHandler --  requiredOperations: [' + originalRequiredOperations + '] -> [' + requiredOperations + ']');
+    }
+
+    if (requiredOperations.length === 0) {
+        return false;
+    }
+
+    if (context.user.admin) { // This handles the getAdminContext() case
+        return true;
+    }
+
+    const roleSpec = config.roles.global[context.user.role];
+    let success = false;
+    if (roleSpec) {
+        for (const requiredOperation of requiredOperations) {
+            if (roleSpec.permissions.includes(requiredOperation)) {
+                success = true;
+                break;
+            }
+        }
+    }
+
+    return success;
+}
+
+function enforceGlobalPermission(context, requiredOperations) {
+    if (!checkGlobalPermission(context, requiredOperations)) {
+        throwPermissionDenied();
+    }
+}
+
+async function _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations) {
+    if (!context.user) {
+        return false;
+    }
+
+    const entityType = entitySettings.getEntityType(entityTypeId);
+
+    if (typeof requiredOperations === 'string') {
+        requiredOperations = [ requiredOperations ];
+    }
+
+    if (context.user.restrictedAccessHandler) {
+        const originalRequiredOperations = requiredOperations;
+        if (context.user.restrictedAccessHandler.permissions) {
+            const entityPerms = context.user.restrictedAccessHandler.permissions[entityTypeId];
+
+            if (!entityPerms) {
+                requiredOperations = [];
+            } else if (entityPerms === true) {
+                // no change to require operations
+            } else if (entityPerms instanceof Set) {
+                requiredOperations = requiredOperations.filter(perm => entityPerms.has(perm));
+            } else {
+                const allowedPerms = entityPerms[entityId];
+                if (allowedPerms) {
+                    requiredOperations = requiredOperations.filter(perm => allowedPerms.has(perm));
+                } else {
+                    requiredOperations = [];
+                }
+            }
+        } else {
+            requiredOperations = [];
+        }
+        log.verbose('check permissions with restrictedAccessHandler --  entityTypeId: ' + entityTypeId + '  entityId: ' + entityId + '  requiredOperations: [' + originalRequiredOperations + '] -> [' + requiredOperations + ']');
+    }
+
+    if (requiredOperations.length === 0) {
+        return false;
+    }
+
+    if (context.user.admin) { // This handles the getAdminContext() case. In this case we don't check the permission, but just the existence.
+        const existsQuery = tx(entityType.entitiesTable);
+
+        if (entityId) {
+            existsQuery.where('id', entityId);
+        }
+
+        const exists = await existsQuery.first();
+
+        return !!exists;
+
+    } else {
+        const permsQuery = tx(entityType.permissionsTable)
+            .where('user', context.user.id)
+            .whereIn('operation', requiredOperations);
+
+        if (entityId) {
+            permsQuery.andWhere('entity', entityId);
+        }
+
+        const perms = await permsQuery.first();
+
+        return !!perms;
+    }
+}
+
+async function checkEntityPermission(context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        return false;
+    }
+
+    return await knex.transaction(async tx => {
+        return await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+    });
+}
+
+async function checkEntityPermissionTx(tx, context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        return false;
+    }
+
+    return await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+}
+
+async function checkTypePermission(context, entityTypeId, requiredOperations) {
+    return await knex.transaction(async tx => {
+        return await _checkPermissionTx(tx, context, entityTypeId, null, requiredOperations);
+    });
+}
+
+async function enforceEntityPermission(context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        throwPermissionDenied();
+    }
+    await knex.transaction(async tx => {
+        const result = await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+        if (!result) {
+            log.info(`Denying permission ${entityTypeId}.${entityId} ${requiredOperations}`);
+            throwPermissionDenied();
+        }
+    });
+}
+
+async function enforceEntityPermissionTx(tx, context, entityTypeId, entityId, requiredOperations) {
+    if (!entityId) {
+        throwPermissionDenied();
+    }
+    const result = await _checkPermissionTx(tx, context, entityTypeId, entityId, requiredOperations);
+    if (!result) {
+        log.info(`Denying permission ${entityTypeId}.${entityId} ${requiredOperations}`);
+        throwPermissionDenied();
+    }
+}
+
+async function enforceTypePermission(context, entityTypeId, requiredOperations) {
+    await knex.transaction(async tx => {
+        const result = await _checkPermissionTx(tx, context, entityTypeId, null, requiredOperations);
+        if (!result) {
+            log.info(`Denying permission ${entityTypeId} ${requiredOperations}`);
+            throwPermissionDenied();
+        }
+    });
+}
+
+async function enforceTypePermissionTx(tx, context, entityTypeId, requiredOperations) {
+    const result = await _checkPermissionTx(tx, context, entityTypeId, null, requiredOperations);
+    if (!result) {
+        log.info(`Denying permission ${entityTypeId} ${requiredOperations}`);
+        throwPermissionDenied();
+    }
+}
+
+function getGlobalPermissions(context) {
+    if (!context.user) {
+        return [];
+    }
+
+    enforce(!context.user.admin, 'getPermissions is not supposed to be called by assumed admin');
+
+    return (config.roles.global[context.user.role] || {}).permissions || [];
+}
+
+async function getPermissionsTx(tx, context, entityTypeId, entityId) {
+    if (!context.user) {
+        return [];
+    }
+
+    enforce(!context.user.admin, 'getPermissions is not supposed to be called by assumed admin');
+
+    const entityType = entitySettings.getEntityType(entityTypeId);
+
+    const rows = await tx(entityType.permissionsTable)
+        .select('operation')
+        .where('entity', entityId)
+        .where('user', context.user.id);
+
+    return rows.map(x => x.operation);
+}
+
+module.exports.listByEntityDTAjax = listByEntityDTAjax;
+module.exports.listByUserDTAjax = listByUserDTAjax;
+module.exports.listUnassignedUsersDTAjax = listUnassignedUsersDTAjax;
+module.exports.listRolesDTAjax = listRolesDTAjax;
+module.exports.assign = assign;
+module.exports.rebuildPermissionsTx = rebuildPermissionsTx;
+module.exports.rebuildPermissions = rebuildPermissions;
+module.exports.removeDefaultShares = removeDefaultShares;
+module.exports.enforceEntityPermission = enforceEntityPermission;
+module.exports.enforceEntityPermissionTx = enforceEntityPermissionTx;
+module.exports.enforceTypePermission = enforceTypePermission;
+module.exports.enforceTypePermissionTx = enforceTypePermissionTx;
+module.exports.checkEntityPermissionTx = checkEntityPermissionTx;
+module.exports.checkEntityPermission = checkEntityPermission;
+module.exports.checkTypePermission = checkTypePermission;
+module.exports.enforceGlobalPermission = enforceGlobalPermission;
+module.exports.checkGlobalPermission = checkGlobalPermission;
+module.exports.throwPermissionDenied = throwPermissionDenied;
+module.exports.regenerateRoleNamesTable = regenerateRoleNamesTable;
+module.exports.getGlobalPermissions = getGlobalPermissions;
+module.exports.getPermissionsTx = getPermissionsTx;

server/models/subscriptions.js

@@ -0,0 +1,826 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const shortid = require('shortid');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const fields = require('./fields');
+const { SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
+const segments = require('./segments');
+const { enforce, filterObject } = require('../lib/helpers');
+const moment = require('moment');
+const { formatDate, formatBirthday } = require('../../shared/date');
+const crypto = require('crypto');
+const campaigns = require('./campaigns');
+const lists = require('./lists');
+
+const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']);
+
+const fieldTypes = {};
+
+const Cardinality = {
+    SINGLE: 0,
+    MULTIPLE: 1
+};
+
+function getOptionsMap(groupedField) {
+    const result = {};
+    for (const opt of groupedField.settings.options) {
+        result[opt.key] = opt.label;
+    }
+
+    return result;
+}
+
+fieldTypes.text = fieldTypes.website = fieldTypes.longtext = fieldTypes.gpg = fieldTypes.number = fieldTypes.json = {
+    afterJSON: (groupedField, entity) => {},
+    listRender: (groupedField, value) => value
+};
+
+fieldTypes['checkbox-grouped'] = {
+    afterJSON: (groupedField, entity) => {},
+    listRender: (groupedField, value) => {
+        const optMap = getOptionsMap(groupedField);
+        return value.map(x => optMap[x]).join(', ');
+    }
+};
+
+fieldTypes['radio-enum'] = fieldTypes['dropdown-enum'] = fieldTypes['radio-grouped'] = fieldTypes['dropdown-grouped'] = {
+    afterJSON: (groupedField, entity) => {},
+    listRender: (groupedField, value) => {
+        const optMap = getOptionsMap(groupedField);
+        return optMap[value];
+    }
+};
+
+fieldTypes.date = {
+    afterJSON: (groupedField, entity) => {
+        const key = getFieldColumn(groupedField);
+        if (key in entity) {
+            entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
+        }
+    },
+    listRender: (groupedField, value) => formatDate(groupedField.settings.dateFormat, value)
+};
+
+fieldTypes.birthday = {
+    afterJSON: (groupedField, entity) => {
+        const key = getFieldColumn(groupedField);
+        if (key in entity) {
+            entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
+        }
+    },
+    listRender: (groupedField, value) => formatBirthday(groupedField.settings.dateFormat, value)
+};
+
+
+
+function getSubscriptionTableName(listId) {
+    return `subscription__${listId}`;
+}
+
+async function getGroupedFieldsMapTx(tx, listId) {
+    const groupedFields = await fields.listGroupedTx(tx, listId);
+    const result = {};
+    for (const fld of groupedFields) {
+        result[getFieldColumn(fld)] = fld;
+    }
+    return result;
+}
+
+function groupSubscription(groupedFieldsMap, entity) {
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+        const fieldType = fields.getFieldType(fld.type);
+
+        if (fieldType.grouped) {
+
+            let value = null;
+
+            if (fieldType.cardinality === fields.Cardinality.SINGLE) {
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+
+                    if (entity[option.column]) {
+                        value = option.column;
+                    }
+
+                    delete entity[option.column];
+                }
+
+            } else {
+                value = [];
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+
+                    if (entity[option.column]) {
+                        value.push(option.column);
+                    }
+
+                    delete entity[option.column];
+                }
+            }
+
+            entity[fldCol] = value;
+
+        } else if (fieldType.enumerated) {
+            // This is enum-xxx type. We just make sure that the options we give out match the field settings.
+            // If the field settings gets changed, there can be discrepancies between the field and the subscription data.
+
+            const allowedKeys = new Set(fld.settings.options.map(x => x.key));
+
+            if (!allowedKeys.has(entity[fldCol])) {
+                entity[fldCol] = null;
+            }
+        }
+    }
+}
+
+function ungroupSubscription(groupedFieldsMap, entity) {
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+
+        const fieldType = fields.getFieldType(fld.type);
+        if (fieldType.grouped) {
+
+            if (fieldType.cardinality === fields.Cardinality.SINGLE) {
+                const value = entity[fldCol];
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+                    entity[option.column] = option.column === value;
+                }
+
+            } else {
+                const values = entity[fldCol] || []; // The default (empty array) is here because create may be called with an entity that has some fields not filled in
+                for (const optionKey in fld.groupedOptions) {
+                    const option = fld.groupedOptions[optionKey];
+                    entity[option.column] = values.includes(option.column);
+                }
+            }
+
+            delete entity[fldCol];
+
+        } else if (fieldType.enumerated) {
+            // This is enum-xxx type. We just make sure that the options we give out match the field settings.
+            // If the field settings gets changed, there can be discrepancies between the field and the subscription data.
+
+            const allowedKeys = new Set(fld.settings.options.map(x => x.key));
+
+            if (!allowedKeys.has(entity[fldCol])) {
+                entity[fldCol] = null;
+            }
+        }
+    }
+}
+
+
+function getAllowedKeys(groupedFieldsMap) {
+    return new Set([
+        ...allowedKeysBase,
+        ...Object.keys(groupedFieldsMap)
+    ]);
+}
+
+function hashByAllowedKeys(allowedKeys, entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function hashByList(listId, entity) {
+    return await knex.transaction(async tx => {
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        const allowedKeys = getAllowedKeys(groupedFieldsMap);
+        return hashByAllowedKeys(allowedKeys, entity);
+    });
+}
+
+async function _getByTx(tx, context, listId, key, value, grouped) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+    const entity = await tx(getSubscriptionTableName(listId)).where(key, value).first();
+
+    if (!entity) {
+        throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+    }
+
+    const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+    if (grouped) {
+        groupSubscription(groupedFieldsMap, entity);
+    }
+
+    return entity;
+}
+
+async function _getBy(context, listId, key, value, grouped) {
+    return await knex.transaction(async tx => {
+        return _getByTx(tx, context, listId, key, value, grouped);
+    });
+}
+
+async function getById(context, listId, id, grouped = true) {
+    return await _getBy(context, listId, 'id', id, grouped);
+}
+
+async function getByEmail(context, listId, email, grouped = true) {
+    return await _getBy(context, listId, 'email', email, grouped);
+}
+
+async function getByCid(context, listId, cid, grouped = true) {
+    return await _getBy(context, listId, 'cid', cid, grouped);
+}
+
+async function getByCidTx(tx, context, listId, cid, grouped = true) {
+    return await _getByTx(tx, context, listId, 'cid', cid, grouped);
+}
+
+async function listDTAjax(context, listId, segmentId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+        const listTable = getSubscriptionTableName(listId);
+
+        // All the data transformation below is to reuse ajaxListTx and groupSubscription methods so as to keep the code DRY
+        // We first construct the columns to contain all which is supposed to be show and extraColumns which contain
+        // everything else that constitutes the subscription.
+        // Then in ajaxList's mapFunc, we construct the entity from the fields ajaxList retrieved and pass it to groupSubscription
+        // to group the fields. Then we copy relevant values form grouped subscription to ajaxList's data which then get
+        // returned to the client. During the copy, we also render the values.
+
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        const listFlds = await fields.listByOrderListTx(tx, listId, ['column', 'id']);
+
+        const columns = [
+            listTable + '.id',
+            listTable + '.cid',
+            listTable + '.email',
+            listTable + '.status',
+            listTable + '.created',
+            { name: 'blacklisted', raw: 'not isnull(blacklist.email)' }
+        ];
+        const extraColumns = [];
+        let listFldIdx = columns.length;
+        const idxMap = {};
+
+        for (const listFld of listFlds) {
+            const fldCol = getFieldColumn(listFld);
+            const fld = groupedFieldsMap[fldCol];
+
+            if (fld.column) {
+                columns.push(listTable + '.' + fld.column);
+            } else {
+                columns.push({
+                    name: listTable + '.' + fldCol,
+                    raw: 0
+                })
+            }
+
+            idxMap[fldCol] = listFldIdx;
+            listFldIdx += 1;
+        }
+
+        for (const fldCol in groupedFieldsMap) {
+            const fld = groupedFieldsMap[fldCol];
+
+            if (fld.column) {
+                if (!(fldCol in idxMap)) {
+                    extraColumns.push(listTable + '.' + fld.column);
+                    idxMap[fldCol] = listFldIdx;
+                    listFldIdx += 1;
+                }
+
+            } else {
+                for (const optionColumn in fld.groupedOptions) {
+                    extraColumns.push(listTable + '.' + optionColumn);
+                    idxMap[optionColumn] = listFldIdx;
+                    listFldIdx += 1;
+                }
+            }
+        }
+
+        const addSegmentQuery = segmentId ? await segments.getQueryGeneratorTx(tx, listId, segmentId) : () => {};
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => {
+                const query = builder
+                    .from(listTable)
+                    .leftOuterJoin('blacklist', listTable + '.email', 'blacklist.email')
+                ;
+                query.where(function() {
+                    addSegmentQuery(this);
+                });
+
+                return query;
+            },
+            columns,
+            {
+                mapFun: data => {
+                    const entity = {};
+                    for (const fldCol in idxMap) {
+                        // This is a bit of hacking. We rely on the fact that if a field has a column, then the column is the field key.
+                        // Then it has the group id with value 0. groupSubscription will be able to process the fields that have a column
+                        // and it will assign values to the fields that don't have a value (i.e. those that currently have the group id and value 0).
+                        entity[fldCol] = data[idxMap[fldCol]];
+                    }
+
+                    groupSubscription(groupedFieldsMap, entity);
+
+                    for (const listFld of listFlds) {
+                        const fldCol = getFieldColumn(listFld);
+                        const fld = groupedFieldsMap[fldCol];
+                        data[idxMap[fldCol]] = fieldTypes[fld.type].listRender(fld, entity[fldCol]);
+                    }
+                },
+
+                extraColumns
+            }
+        );
+    });
+}
+
+async function listTestUsersDTAjax(context, listCid, params) {
+    return await knex.transaction(async tx => {
+        const list = await lists.getByCidTx(tx, context, listCid);
+        await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
+
+        const listTable = getSubscriptionTableName(list.id);
+
+        const columns = [
+            listTable + '.id',
+            listTable + '.cid',
+            listTable + '.email',
+            listTable + '.status',
+            listTable + '.created'
+        ];
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from(listTable)
+                .where('is_test', true),
+            columns,
+            {}
+        );
+    });
+}
+
+async function list(context, listId, grouped = true, offset, limit) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+        const count = await tx(getSubscriptionTableName(listId)).count('* as count').first().count;
+
+        const entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc');
+
+        if (Number.isInteger(offset)) {
+            entitiesQry.offset(offset);
+        }
+
+        if (Number.isInteger(limit)) {
+            entitiesQry.limit(limit);
+        }
+
+        const entities = await entitiesQry;
+
+        if (grouped) {
+            const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+            for (const entity of entities) {
+                groupSubscription(groupedFieldsMap, entity);
+            }
+        }
+
+        return {
+            subscriptions: entities,
+            total: count
+        };
+    });
+}
+
+// Note that this does not do all the work in the transaction. Thus it is prone to fail if the list is deleted in during the run of the function
+async function* listIterator(context, listId, segmentId, grouped = true) {
+    let groupedFieldsMap;
+    let addSegmentQuery;
+
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
+
+        if (grouped) {
+            groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        }
+
+        addSegmentQuery = segmentId ? await segments.getQueryGeneratorTx(tx, listId, segmentId) : () => {};
+    });
+
+    let lastId = 0;
+
+    while (true) {
+        const entities = await knex(getSubscriptionTableName(listId))
+            .orderBy('id', 'asc')
+            .where('id', '>', lastId)
+            .where(function() {
+                addSegmentQuery(this);
+            })
+            .limit(500);
+
+        if (entities.length > 0) {
+            for (const entity of entities) {
+                if (grouped) {
+                    groupSubscription(groupedFieldsMap, entity);
+                }
+
+                yield entity;
+            }
+
+            lastId = entities[entities.length - 1].id;
+        } else {
+            break;
+        }
+    }
+}
+
+async function serverValidate(context, listId, data) {
+    return await knex.transaction(async tx => {
+        const result = {};
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        if (data.email) {
+            const existingKeyQuery = tx(getSubscriptionTableName(listId)).where('email', data.email);
+
+            if (data.id) {
+                existingKeyQuery.whereNot('id', data.id);
+            }
+
+            const existingKey = await existingKeyQuery.first();
+            result.key = {
+                exists: !!existingKey
+            };
+        }
+
+        return result;
+    });
+}
+
+async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) {
+    enforce(entity.email, 'Email must be set');
+
+    const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(entity.email));
+
+    if (!isCreate) {
+        existingWithKeyQuery.whereNot('id', entity.id);
+    }
+    const existingWithKey = await existingWithKeyQuery.first();
+    if (existingWithKey) {
+        if (meta && (meta.updateAllowed || (meta.updateOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED))) {
+            meta.update = true;
+            meta.existing = existingWithKey;
+        } else {
+            throw new interoperableErrors.DuplicitEmailError();
+        }
+    } else {
+        // This is here because of the API endpoint, which allows one to submit subscriptions without caring about whether they already exist, what their status is, etc.
+        // The same for import where we need to subscribed only those (existing and new) that have not been unsubscribed already.
+        // In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status to SUBSCRIBED
+
+        if (meta && meta.subscribeIfNoExisting && !entity.status) {
+            entity.status = SubscriptionStatus.SUBSCRIBED;
+        }
+    }
+
+    if ((isCreate && !(meta && meta.update)) || 'status' in entity) {
+        enforce(entity.status >= SubscriptionStatus.MIN && entity.status <= SubscriptionStatus.MAX, 'Invalid status');
+    }
+
+    for (const key in groupedFieldsMap) {
+        const fld = groupedFieldsMap[key];
+
+        fieldTypes[fld.type].afterJSON(fld, entity);
+    }
+}
+
+function hashEmail(email) {
+    return crypto.createHash('sha512').update(email).digest("base64");
+}
+
+function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
+    if ('email' in subscription) {
+        subscription.hash_email = hashEmail(subscription.email);
+        subscription.source_email = source;
+    }
+
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+
+        const fieldType = fields.getFieldType(fld.type);
+        if (fieldType.grouped) {
+            for (const optionKey in fld.groupedOptions) {
+                const option = fld.groupedOptions[optionKey];
+
+                if (option.column in subscription) {
+                    subscription['source_' + option.column] = source;
+                }
+            }
+        } else {
+            if (fldCol in subscription) {
+                subscription['source_' + fldCol] = source;
+            }
+        }
+    }
+}
+
+async function _update(tx, listId, existing, filteredEntity) {
+    if ('status' in filteredEntity) {
+        if (existing.status !== filteredEntity.status) {
+            filteredEntity.status_change = new Date();
+        }
+    }
+
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
+
+    if ('status' in filteredEntity) {
+        let countIncrement = 0;
+        if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
+            countIncrement = -1;
+        } else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
+            countIncrement = 1;
+        }
+
+        if (countIncrement) {
+            await tx('lists').where('id', listId).increment('subscribers', countIncrement);
+        }
+    }
+}
+
+async function _create(tx, listId, filteredEntity) {
+    const ids = await tx(getSubscriptionTableName(listId)).insert(filteredEntity);
+    const id = ids[0];
+
+    if (filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).increment('subscribers', 1);
+    }
+
+    return id;
+}
+
+/*
+    Adds a new subscription. Returns error if a subscription with the same email address is already present and is not unsubscribed.
+    If it is unsubscribed and meta.updateOfUnsubscribedAllowed, the existing subscription is changed based on the provided data.
+    If meta.updateAllowed is true, it updates even an active subscription.
+ */
+async function createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+    const allowedKeys = getAllowedKeys(groupedFieldsMap);
+
+    await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, true);
+
+    const filteredEntity = filterObject(entity, allowedKeys);
+    filteredEntity.status_change = new Date();
+
+    ungroupSubscription(groupedFieldsMap, filteredEntity);
+
+    updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
+
+    filteredEntity.opt_in_ip = meta && meta.ip;
+    filteredEntity.opt_in_country = meta && meta.country;
+
+    if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
+        await _update(tx, listId, meta.existing, filteredEntity);
+        meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
+        return meta.existing.id;
+    } else {
+        filteredEntity.cid = shortid.generate();
+
+        if (meta) {
+            meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
+        }
+
+        return await _create(tx, listId, filteredEntity);
+    }
+}
+
+async function create(context, listId, entity, source, meta) {
+    return await knex.transaction(async tx => {
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        return await createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta);
+    });
+}
+
+async function updateWithConsistencyCheck(context, listId, entity, source) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+        const allowedKeys = getAllowedKeys(groupedFieldsMap);
+
+        groupSubscription(groupedFieldsMap, existing);
+
+        const existingHash = hashByAllowedKeys(allowedKeys, existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, null, false);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+
+        ungroupSubscription(groupedFieldsMap, filteredEntity);
+
+        updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
+
+        await _update(tx, listId, existing, filteredEntity);
+    });
+}
+
+async function _removeAndGetTx(tx, context, listId, existing) {
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).del();
+
+    if (existing.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).decrement('subscribers', 1);
+    }
+}
+
+async function remove(context, listId, id) {
+    await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', id).first();
+        await _removeAndGetTx(tx, context, listId, existing);
+    });
+}
+
+async function removeByEmailAndGet(context, listId, email) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
+        return await _removeAndGetTx(tx, context, listId, existing);
+    });
+}
+
+
+async function _changeStatusTx(tx, context, listId, existing, newStatus) {
+    enforce(newStatus !== SubscriptionStatus.SUBSCRIBED);
+
+    await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update({
+        status: newStatus
+    });
+
+    if (existing.status === SubscriptionStatus.SUBSCRIBED) {
+        await tx('lists').where('id', listId).decrement('subscribers', 1);
+    }
+}
+
+async function _unsubscribeExistingAndGetTx(tx, context, listId, existing) {
+    if (!(existing && existing.status === SubscriptionStatus.SUBSCRIBED)) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    await _changeStatusTx(tx, context, listId, existing, SubscriptionStatus.UNSUBSCRIBED);
+
+    existing.status = SubscriptionStatus.SUBSCRIBED;
+
+    return existing;
+}
+
+async function unsubscribeByIdAndGet(context, listId, subscriptionId) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+        return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
+    });
+}
+
+async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaignCid) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('cid', subscriptionCid).first();
+
+        if (campaignCid) {
+            await campaigns.changeStatusByCampaignCidAndSubscriptionIdTx(tx, context, campaignCid, listId, existing.id, SubscriptionStatus.UNSUBSCRIBED);
+        }
+
+        return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
+    });
+}
+
+async function unsubscribeByEmailAndGetTx(tx, context, listId, email) {
+    const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
+    return await _unsubscribeExistingAndGetTx(tx, context, listId, existing);
+}
+
+async function unsubscribeByEmailAndGet(context, listId, email) {
+    return await knex.transaction(async tx => {
+        return await unsubscribeByEmailAndGetTx(tx, context, listId, email);
+    });
+}
+
+async function changeStatusTx(tx, context, listId, subscriptionId, subscriptionStatus) {
+    const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+    await _changeStatusTx(tx, context, listId, existing, subscriptionStatus);
+}
+
+
+
+async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (existing.email !== emailNew) {
+            await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(emailNew)).del();
+
+            await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
+                email: emailNew
+            });
+
+            existing.email = emailNew;
+        }
+
+        return existing;
+    });
+}
+
+async function updateManaged(context, listId, cid, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
+
+        const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
+        const update = {};
+        for (const key in groupedFieldsMap) {
+            const fld = groupedFieldsMap[key];
+
+            if (fld.order_manage) {
+                update[key] = entity[key];
+            }
+
+            fieldTypes[fld.type].afterJSON(fld, update);
+        }
+
+        ungroupSubscription(groupedFieldsMap, update);
+
+        await tx(getSubscriptionTableName(listId)).where('cid', cid).update(update);
+    });
+}
+
+
+async function getListsWithEmail(context, email) {
+    // FIXME - this methods is rather suboptimal if there are many lists. It quite needs permission caching in shares.js
+
+    return await knex.transaction(async tx => {
+        const lsts = await tx('lists').select(['id', 'name']);
+        const result = [];
+
+        for (const list of lsts) {
+            await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
+            const entity = await tx(getSubscriptionTableName(list.id)).where('email', email).first();
+            if (entity) {
+                result.push(list);
+            }
+        }
+
+        return result;
+    });
+}
+
+module.exports.getSubscriptionTableName = getSubscriptionTableName;
+module.exports.hashByList = hashByList;
+module.exports.getById = getById;
+module.exports.getByCidTx = getByCidTx;
+module.exports.getByCid = getByCid;
+module.exports.getByEmail = getByEmail;
+module.exports.list = list;
+module.exports.listIterator = listIterator;
+module.exports.listDTAjax = listDTAjax;
+module.exports.listTestUsersDTAjax = listTestUsersDTAjax;
+module.exports.serverValidate = serverValidate;
+module.exports.create = create;
+module.exports.getGroupedFieldsMapTx = getGroupedFieldsMapTx;
+module.exports.createTxWithGroupedFieldsMap = createTxWithGroupedFieldsMap;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;
+module.exports.removeByEmailAndGet = removeByEmailAndGet;
+module.exports.unsubscribeByCidAndGet = unsubscribeByCidAndGet;
+module.exports.unsubscribeByIdAndGet = unsubscribeByIdAndGet;
+module.exports.unsubscribeByEmailAndGet = unsubscribeByEmailAndGet;
+module.exports.unsubscribeByEmailAndGetTx = unsubscribeByEmailAndGetTx;
+module.exports.updateAddressAndGet = updateAddressAndGet;
+module.exports.updateManaged = updateManaged;
+module.exports.getListsWithEmail = getListsWithEmail;
+module.exports.changeStatusTx = changeStatusTx;

server/models/templates.js

@@ -0,0 +1,123 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const namespaceHelpers = require('../lib/namespace-helpers');
+const shares = require('./shares');
+const reports = require('./reports');
+const files = require('./files');
+const dependencyHelpers = require('../lib/dependency-helpers');
+
+const allowedKeys = new Set(['name', 'description', 'type', 'data', 'html', 'text', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getByIdTx(tx, context, id, withPermissions = true) {
+    await shares.enforceEntityPermissionTx(tx, context, 'template', id, 'view');
+    const entity = await tx('templates').where('id', id).first();
+    entity.data = JSON.parse(entity.data);
+
+    if (withPermissions) {
+        entity.permissions = await shares.getPermissionsTx(tx, context, 'template', id);
+    }
+
+    return entity;
+}
+
+async function getById(context, id, withPermissions = true) {
+    return await knex.transaction(async tx => {
+        return await getByIdTx(tx, context, id, withPermissions);
+    });
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'template', requiredOperations: ['view'] }],
+        params,
+        builder => builder.from('templates').innerJoin('namespaces', 'namespaces.id', 'templates.namespace'),
+        [ 'templates.id', 'templates.name', 'templates.description', 'templates.type', 'templates.created', 'namespaces.name' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, entity) {
+    await namespaceHelpers.validateEntity(tx, entity);
+
+    // We don't check contents of the "data" because it is processed solely on the client. The client generates the HTML code we use when sending out campaigns.
+
+    entity.data = JSON.stringify(entity.data);
+}
+
+async function create(context, entity) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createTemplate');
+
+        await _validateAndPreprocess(tx, entity);
+
+        const ids = await tx('templates').insert(filterObject(entity, allowedKeys));
+        const id = ids[0];
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'template', entityId: id });
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'template', entity.id, 'edit');
+
+        const existing = await tx('templates').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        existing.data = JSON.parse(existing.data);
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, entity);
+
+        await namespaceHelpers.validateMove(context, entity, existing, 'template', 'createTemplate', 'delete');
+
+        await tx('templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissionsTx(tx, { entityTypeId: 'template', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'template', id, 'delete');
+
+        await dependencyHelpers.ensureNoDependencies(tx, context, id, [
+            {
+                entityTypeId: 'campaign',
+                query: tx => tx('template_dep_campaigns')
+                    .where('template_dep_campaigns.template', id)
+                    .innerJoin('campaigns', 'template_dep_campaigns.campaign', 'campaigns.id')
+                    .select(['campaigns.id', 'campaigns.name'])
+            }
+        ]);
+
+        await files.removeAllTx(tx, context, 'template', 'file', id);
+
+        await tx('templates').where('id', id).del();
+    });
+}
+
+module.exports.hash = hash;
+module.exports.getByIdTx = getByIdTx;
+module.exports.getById = getById;
+module.exports.listDTAjax = listDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.remove = remove;

server/models/triggers.js

@@ -0,0 +1,145 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const shares = require('./shares');
+const {EntityVals, EventVals, Entity} = require('../../shared/triggers');
+const campaigns = require('./campaigns');
+
+const allowedKeys = new Set(['name', 'description', 'entity', 'event', 'seconds', 'enabled', 'source_campaign']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
+}
+
+async function getById(context, campaignId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'viewTriggers');
+
+        const entity = await tx('triggers').where({campaign: campaignId, id}).first();
+
+        return entity;
+    });
+}
+
+async function listByCampaignDTAjax(context, campaignId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'viewTriggers');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('triggers')
+                .innerJoin('campaigns', 'campaigns.id', 'triggers.campaign')
+                .where('triggers.campaign', campaignId),
+            [ 'triggers.id', 'triggers.name', 'triggers.description', 'triggers.entity', 'triggers.event', 'triggers.seconds', 'triggers.enabled' ]
+        );
+    });
+}
+
+async function listByListDTAjax(context, listId, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'campaign', requiredOperations: ['viewTriggers'] }],
+        params,
+        builder => builder
+            .from('triggers')
+            .innerJoin('campaigns', 'campaigns.id', 'triggers.campaign')
+            .innerJoin('campaign_lists', 'campaign_lists.campaign', 'campaigns.id')
+            .where('campaign_lists.list', listId),
+        [ 'triggers.id', 'triggers.name', 'triggers.description', 'campaigns.name', 'triggers.entity', 'triggers.event', 'triggers.seconds', 'triggers.enabled', 'triggers.campaign' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, context, campaignId, entity) {
+    enforce(Number.isInteger(entity.seconds));
+    enforce(entity.seconds >= 0, 'Seconds must not be negative');
+    enforce(entity.entity in EntityVals, 'Invalid entity');
+    enforce(entity.event in EventVals[entity.entity], 'Invalid event');
+
+    if (entity.entity === Entity.CAMPAIGN) {
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.source_campaign, 'view');
+    }
+
+    await campaigns.enforceSendPermissionTx(tx, context, campaignId);
+}
+
+async function create(context, campaignId, entity) {
+    return await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'manageTriggers');
+
+        await _validateAndPreprocess(tx, context, campaignId, entity);
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.campaign = campaignId;
+
+        const ids = await tx('triggers').insert(filteredEntity);
+        const id = ids[0];
+
+        return id;
+    });
+}
+
+async function updateWithConsistencyCheck(context, campaignId, entity) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'manageTriggers');
+
+        const existing = await tx('triggers').where({campaign: campaignId, id: entity.id}).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await _validateAndPreprocess(tx, context, campaignId, entity);
+
+        await tx('triggers').where({campaign: campaignId, id: entity.id}).update(filterObject(entity, allowedKeys));
+    });
+}
+
+async function removeTx(tx, context, campaignId, id) {
+    await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'manageTriggers');
+
+    const existing = await tx('triggers').where({campaign: campaignId, id}).first();
+    if (!existing) {
+        throw new interoperableErrors.NotFoundError();
+    }
+
+    await tx('trigger_messages').where({trigger: id}).del();
+    await tx('triggers').where('id', id).del();
+}
+
+async function remove(context, campaignId, id) {
+    await knex.transaction(async tx => {
+        await removeTx(tx, context, campaignId, id);
+    });
+}
+
+async function removeAllByCampaignIdTx(tx, context, campaignId) {
+    const entities = await tx('triggers').where('campaign', campaignId).select(['id']);
+    for (const entity of entities) {
+        await removeTx(tx, context, campaignId, entity.id);
+    }
+}
+
+
+
+// This is to handle circular dependency with campaigns.js
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.listByCampaignDTAjax = listByCampaignDTAjax;
+module.exports.listByListDTAjax = listByListDTAjax;
+module.exports.create = create;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.removeTx = removeTx;
+module.exports.remove = remove;
+module.exports.removeAllByCampaignIdTx = removeAllByCampaignIdTx;

server/models/users.js

@@ -0,0 +1,439 @@
+'use strict';
+
+const config = require('config');
+const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
+const { enforce, filterObject } = require('../lib/helpers');
+const interoperableErrors = require('../../shared/interoperable-errors');
+const passwordValidator = require('../../shared/password-validator')();
+const dtHelpers = require('../lib/dt-helpers');
+const tools = require('../lib/tools');
+const crypto = require('crypto');
+const settings = require('./settings');
+const {getTrustedUrl} = require('../lib/urls');
+const { tUI } = require('../lib/translate');
+
+const bluebird = require('bluebird');
+
+const bcrypt = require('bcrypt-nodejs');
+const bcryptHash = bluebird.promisify(bcrypt.hash);
+const bcryptCompare = bluebird.promisify(bcrypt.compare);
+
+const mailers = require('../lib/mailers');
+
+const passport = require('../lib/passport');
+
+const namespaceHelpers = require('../lib/namespace-helpers');
+
+const allowedKeys = new Set(['username', 'name', 'email', 'password', 'namespace', 'role']);
+const ownAccountAllowedKeys = new Set(['name', 'email', 'password']);
+const allowedKeysExternal = new Set(['username', 'namespace', 'role']);
+const hashKeys = new Set(['username', 'name', 'email', 'namespace', 'role']);
+const shares = require('./shares');
+const contextHelpers = require('../lib/context-helpers');
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function _getBy(context, key, value, extraColumns = []) {
+    const columns = ['id', 'username', 'name', 'email', 'namespace', 'role', ...extraColumns];
+
+    const user = await knex('users').select(columns).where(key, value).first();
+
+    if (!user) {
+        shares.throwPermissionDenied();
+    }
+
+    await shares.enforceEntityPermission(context, 'namespace', user.namespace, 'manageUsers');
+
+    return user;
+}
+
+async function getById(context, id) {
+    return await _getBy(context, 'id', id);
+}
+
+async function serverValidate(context, data, isOwnAccount) {
+    const result = {};
+
+    if (!isOwnAccount) {
+        await shares.enforceTypePermission(context, 'namespace', 'manageUsers');
+    }
+
+    if (!isOwnAccount && data.username) {
+        const query = knex('users').select(['id']).where('username', data.username);
+
+        if (data.id) {
+            // Id is not set in entity creation form
+            query.andWhereNot('id', data.id);
+        }
+
+        const user = await query.first();
+        result.username = {
+            exists: !!user
+        };
+    }
+
+    if (isOwnAccount && data.currentPassword) {
+        const user = await knex('users').select(['id', 'password']).where('id', data.id).first();
+
+        result.currentPassword = {};
+        result.currentPassword.incorrect = !await bcryptCompare(data.currentPassword, user.password);
+    }
+
+    if (data.email) {
+        const query = knex('users').select(['id']).where('email', data.email);
+
+        if (data.id) {
+            // Id is not set in entity creation form
+            query.andWhereNot('id', data.id);
+        }
+
+        const user = await query.first();
+
+        result.email = {};
+        result.email.invalid = await tools.validateEmail(data.email) !== 0;
+        result.email.exists = !!user;
+    }
+
+    return result;
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'namespace', requiredOperations: ['manageUsers'] }],
+        params,
+        builder => builder
+            .from('users')
+            .innerJoin('namespaces', 'namespaces.id', 'users.namespace')
+            .innerJoin('generated_role_names', 'generated_role_names.role', 'users.role')
+            .where('generated_role_names.entity_type', 'global'),
+        [ 'users.id', 'users.username', 'users.name', 'namespaces.name', 'generated_role_names.name' ]
+    );
+}
+
+async function _validateAndPreprocess(tx, entity, isCreate, isOwnAccount) {
+    enforce(await tools.validateEmail(entity.email) === 0, 'Invalid email');
+
+    await namespaceHelpers.validateEntity(tx, entity);
+
+    const otherUserWithSameEmailQuery = tx('users').where('email', entity.email);
+    if (entity.id) {
+        otherUserWithSameEmailQuery.andWhereNot('id', entity.id);
+    }
+
+    if (await otherUserWithSameEmailQuery.first()) {
+        throw new interoperableErrors.DuplicitEmailError();
+    }
+
+
+    if (!isOwnAccount) {
+        const otherUserWithSameUsernameQuery = tx('users').where('username', entity.username);
+        if (entity.id) {
+            otherUserWithSameUsernameQuery.andWhereNot('id', entity.id);
+        }
+
+        if (await otherUserWithSameUsernameQuery.first()) {
+            throw new interoperableErrors.DuplicitNameError();
+        }
+    }
+
+    enforce(entity.role in config.roles.global, 'Unknown role');
+
+    enforce(!isCreate || entity.password.length > 0, 'Password not set');
+
+    if (entity.password) {
+        const passwordValidatorResults = passwordValidator.test(entity.password);
+        if (passwordValidatorResults.errors.length > 0) {
+            // This is not an interoperable error because this is not supposed to happen unless the client is tampered with.
+            throw new Error('Invalid password');
+        }
+
+        entity.password = await bcryptHash(entity.password, null, null);
+    } else {
+        delete entity.password;
+    }
+}
+
+async function create(context, user) {
+    let id;
+    await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', user.namespace, 'manageUsers');
+
+        if (passport.isAuthMethodLocal) {
+            await _validateAndPreprocess(tx, user, true);
+
+            const ids = await tx('users').insert(filterObject(user, allowedKeys));
+            id = ids[0];
+
+        } else {
+            const filteredUser = filterObject(user, allowedKeysExternal);
+            enforce(user.role in config.roles.global, 'Unknown role');
+
+            await namespaceHelpers.validateEntity(tx, user);
+
+            const ids = await tx('users').insert(filteredUser);
+            id = ids[0];
+        }
+
+        await shares.rebuildPermissionsTx(tx, { userId: id });
+    });
+
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, user, isOwnAccount) {
+    await knex.transaction(async tx => {
+        const existing = await tx('users').where('id', user.id).first();
+        if (!existing) {
+            shares.throwPermissionDenied();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash !== user.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        if (!isOwnAccount) {
+            await shares.enforceEntityPermissionTx(tx, context, 'namespace', user.namespace, 'manageUsers');
+            await shares.enforceEntityPermissionTx(tx, context, 'namespace', existing.namespace, 'manageUsers');
+        }
+
+        if (passport.isAuthMethodLocal) {
+            await _validateAndPreprocess(tx, user, false, isOwnAccount);
+
+            if (isOwnAccount && user.password) {
+                if (!await bcryptCompare(user.currentPassword, existing.password)) {
+                    throw new interoperableErrors.IncorrectPasswordError();
+                }
+            }
+
+            await tx('users').where('id', user.id).update(filterObject(user, isOwnAccount ? ownAccountAllowedKeys : allowedKeys));
+        } else {
+            enforce(!isOwnAccount, 'Local user management is required');
+            enforce(user.role in config.roles.global, 'Unknown role');
+            await namespaceHelpers.validateEntity(tx, user);
+
+            await tx('users').where('id', user.id).update(filterObject(user, allowedKeysExternal));
+        }
+
+        // Removes the default shares based on the user role and rebuilds permissions.
+        // rebuildPermissions adds the default shares based on the user role, which will reflect the changes
+        // done to the user.
+        if (existing.namespace !== user.namespace || existing.role !== user.role) {
+            await shares.removeDefaultShares(tx, existing);
+        }
+
+        await shares.rebuildPermissionsTx(tx, { userId: user.id });
+    });
+}
+
+async function remove(context, userId) {
+    enforce(userId !== 1, 'Admin cannot be deleted');
+    enforce(context.user.id !== userId, 'User cannot delete himself/herself');
+
+    await knex.transaction(async tx => {
+        const existing = await tx('users').where('id', userId).first();
+        if (!existing) {
+            shares.throwPermissionDenied();
+        }
+
+        await shares.enforceEntityPermissionTx(tx, context, 'namespace', existing.namespace, 'manageUsers');
+
+        await tx('users').where('id', userId).del();
+    });
+}
+
+async function getByAccessToken(accessToken) {
+    return await _getBy(contextHelpers.getAdminContext(), 'access_token', accessToken);
+}
+
+async function getByUsername(username) {
+    return await _getBy(contextHelpers.getAdminContext(), 'username', username);
+}
+
+async function getByUsernameIfPasswordMatch(username, password) {
+    try {
+        const user = await _getBy(contextHelpers.getAdminContext(), 'username', username, ['password']);
+
+        if (!await bcryptCompare(password, user.password)) {
+            throw new interoperableErrors.IncorrectPasswordError();
+        }
+
+        delete user.password;
+
+        return user;
+
+    } catch (err) {
+        if (err instanceof interoperableErrors.NotFoundError) {
+            throw new interoperableErrors.IncorrectPasswordError();
+        }
+
+        throw err;
+    }
+}
+
+async function getAccessToken(userId) {
+    const user = await _getBy(contextHelpers.getAdminContext(), 'id', userId, ['access_token']);
+    return user.access_token;
+}
+
+async function resetAccessToken(userId) {
+    const token = crypto.randomBytes(20).toString('hex').toLowerCase();
+    await knex('users').where({id: userId}).update({access_token: token});
+    return token;
+}
+
+async function sendPasswordReset(language, usernameOrEmail) {
+    enforce(passport.isAuthMethodLocal, 'Local user management is required');
+
+    await knex.transaction(async tx => {
+        const user = await tx('users').where('username', usernameOrEmail).orWhere('email', usernameOrEmail).select(['id', 'username', 'email', 'name']).first();
+
+        if (user) {
+            const resetToken = crypto.randomBytes(16).toString('base64').replace(/[^a-z0-9]/gi, '');
+
+            await tx('users').where('id', user.id).update({
+                reset_token: resetToken,
+                reset_expire: new Date(Date.now() + 60 * 60 * 1000)
+            });
+
+            const { adminEmail } = await settings.get(contextHelpers.getAdminContext(), ['adminEmail']);
+
+            const mailer = await mailers.getOrCreateMailer();
+            await mailer.sendTransactionalMail({
+                from: {
+                    address: adminEmail
+                },
+                to: {
+                    address: user.email
+                },
+                subject: tUI('account.passwordChangeRequest', language)
+            }, {
+                html: 'emails/password-reset-html.hbs',
+                text: 'emails/password-reset-text.hbs',
+                data: {
+                    title: 'Mailtrain',
+                    username: user.username,
+                    name: user.name,
+                    confirmUrl: getTrustedUrl(`/account/reset/${encodeURIComponent(user.username)}/${encodeURIComponent(resetToken)}`)
+                }
+            });
+        }
+        // We intentionally silently ignore the situation when user is not found. This is not to reveal if a user exists in the system.
+    });
+}
+
+async function isPasswordResetTokenValid(username, resetToken) {
+    enforce(passport.isAuthMethodLocal, 'Local user management is required');
+
+    const user = await knex('users').select(['id']).where({username, reset_token: resetToken}).andWhere('reset_expire', '>', new Date()).first();
+    return !!user;
+}
+
+async function resetPassword(username, resetToken, password) {
+    enforce(passport.isAuthMethodLocal, 'Local user management is required');
+
+    await knex.transaction(async tx => {
+        const user = await tx('users').select(['id']).where({
+            username,
+            reset_token: resetToken
+        }).andWhere('reset_expire', '>', new Date()).first();
+
+        if (user) {
+            const passwordValidatorResults = passwordValidator.test(password);
+            if (passwordValidatorResults.errors.length > 0) {
+                // This is not an interoperable error because this is not supposed to happen unless the client is tampered with.
+                throw new Error('Invalid password');
+            }
+
+            password = await bcryptHash(password, null, null);
+
+            await tx('users').where({username}).update({
+                password,
+                reset_token: null,
+                reset_expire: null
+            });
+        } else {
+            throw new interoperableErrors.InvalidTokenError();
+        }
+    });
+}
+
+
+
+const restrictedAccessTokenMethods = {};
+const restrictedAccessTokens = new Map();
+
+function registerRestrictedAccessTokenMethod(method, getHandlerFromParams) {
+    restrictedAccessTokenMethods[method] = getHandlerFromParams;
+}
+
+async function getRestrictedAccessToken(context, method, params) {
+    const token = crypto.randomBytes(24).toString('hex').toLowerCase();
+    const tokenEntry = {
+        token,
+        userId: context.user.id,
+        handler: await restrictedAccessTokenMethods[method](params),
+        expires: Date.now() + 120 * 1000
+    };
+
+    restrictedAccessTokens.set(token, tokenEntry);
+
+    return token;
+}
+
+async function refreshRestrictedAccessToken(context, token) {
+    const tokenEntry = restrictedAccessTokens.get(token);
+
+    if (tokenEntry && tokenEntry.userId === context.user.id) {
+        tokenEntry.expires = Date.now() + 120 * 1000
+    } else {
+        shares.throwPermissionDenied();
+    }
+}
+
+async function getByRestrictedAccessToken(token) {
+    const now = Date.now();
+    for (const entry of restrictedAccessTokens.values()) {
+        if (entry.expires < now) {
+            restrictedAccessTokens.delete(entry.token);
+        }
+    }
+
+    const tokenEntry = restrictedAccessTokens.get(token);
+
+    if (tokenEntry) {
+        const user = await getById(contextHelpers.getAdminContext(), tokenEntry.userId);
+        user.restrictedAccessHandler = tokenEntry.handler;
+        user.restrictedAccessToken = tokenEntry.token;
+
+        return user;
+
+    } else {
+        shares.throwPermissionDenied();
+    }
+}
+
+
+module.exports.listDTAjax = listDTAjax;
+module.exports.remove = remove;
+module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck;
+module.exports.create = create;
+module.exports.hash = hash;
+module.exports.getById = getById;
+module.exports.serverValidate = serverValidate;
+module.exports.getByAccessToken = getByAccessToken;
+module.exports.getByUsername = getByUsername;
+module.exports.getByUsernameIfPasswordMatch = getByUsernameIfPasswordMatch;
+module.exports.getAccessToken = getAccessToken;
+module.exports.resetAccessToken = resetAccessToken;
+module.exports.sendPasswordReset = sendPasswordReset;
+module.exports.isPasswordResetTokenValid = isPasswordResetTokenValid;
+module.exports.resetPassword = resetPassword;
+module.exports.getByRestrictedAccessToken = getByRestrictedAccessToken;
+module.exports.getRestrictedAccessToken = getRestrictedAccessToken;
+module.exports.refreshRestrictedAccessToken = refreshRestrictedAccessToken;
+module.exports.registerRestrictedAccessTokenMethod = registerRestrictedAccessTokenMethod;