Fixes in subscriptions. It now passes the tests.

API tests still don't work.

lib/dt-helpers.js

@@ -104,6 +104,7 @@ async function ajaxListTx(tx, params, queryFun, columns, options) {
 }
 
 async function ajaxListWithPermissionsTx(tx, context, fetchSpecs, params, queryFun, columns, options) {
+    // Note that this function is not intended to be used with the synthetic admin context obtained by contextHelpers.getAdminContext()
     options = options || {};
 
     const permCols = [];

lib/helpers.js

@@ -10,7 +10,8 @@ module.exports = {
     getListMergeTags,
     rollbackAndReleaseConnection,
     filterObject,
-    enforce
+    enforce,
+    cleanupFromPost
 };
 
 function getDefaultMergeTags(callback) {
@@ -128,3 +129,7 @@ function enforce(condition, message) {
         throw new Error(message);
     }
 }
+
+function cleanupFromPost(value) {
+    return (value || '').toString().trim();
+}

lib/tools-async.js

@@ -6,13 +6,11 @@ const isemail = require('isemail');
 
 const bluebird = require('bluebird');
 const mergeTemplateIntoLayout = bluebird.promisify(require('./tools').mergeTemplateIntoLayout);
-const queryParams = require('./tools').queryParams;
 
 module.exports = {
     validateEmail,
     validateEmailGetMessage,
-    mergeTemplateIntoLayout,
+    mergeTemplateIntoLayout
-    queryParams
 };
 
 async function validateEmail(address, checkBlocked) {

models/blacklist.js

@@ -16,45 +16,15 @@ async function listDTAjax(context, params) {
     );
 }
 
-/*
+async function search(context, offset, limit, search) {
-module.exports.get = (start, limit, search, callback) => {
-    db.getConnection((err, connection) => {
-        if (err) {
-            return callback(err);
-        }
-        search = '%' + search + '%';
-        connection.query('SELECT SQL_CALC_FOUND_ROWS `email` FROM blacklist WHERE `email` LIKE ? ORDER BY `email` LIMIT ? OFFSET ?', [search, limit, start], (err, rows) => {
-            if (err) {
-                return callback(err);
-            }
-
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                let emails = [];
-                rows.forEach(email => {
-                    emails.push(email.email);
-                });
-                return callback(null, emails, total && total[0] && total[0].total);
-            });
-        });
-    });
-};
-*/
-
-async function search(context, start, limit, search) {
     return await knex.transaction(async tx => {
         shares.enforceGlobalPermission(context, 'manageBlacklist');
 
         search = '%' + search + '%';
 
-        const count = await tx('blacklist').where('email', 'like', search).count();
+        const count = await tx('blacklist').where('email', 'like', search).count('* as count').first().count;
-        // FIXME - the count won't likely work;
-        console.log(count);
 
-        const rows = await tx('blacklist').where('email', 'like', search).offset(start).limit(limit);
+        const rows = await tx('blacklist').where('email', 'like', search).offset(offset).limit(limit);
 
         return {
             emails: rows.map(row => row.email),

models/fields.js

@@ -10,6 +10,10 @@ const shares = require('./shares');
 const validators = require('../shared/validators');
 const shortid = require('shortid');
 const segments = require('./segments');
+const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../shared/date');
+const { getFieldKey } = require('../shared/lists');
+const { cleanupFromPost } = require('../lib/helpers');
+
 
 const allowedKeysCreate = new Set(['name', 'key', 'default_value', 'type', 'group', 'settings']);
 const allowedKeysUpdate = new Set(['name', 'key', 'default_value', 'group', 'settings']);
@@ -22,72 +26,133 @@ const Cardinality = {
     MULTIPLE: 1
 };
 
-fieldTypes.text = fieldTypes.website = {
+fieldTypes.text = {
-    validate: entity => {},
+    validate: field => {},
     addColumn: (table, name) => table.string(name),
     indexed: true,
     grouped: false,
     enumerated: false,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeText',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value
 };
 
-fieldTypes.longtext = fieldTypes.gpg = {
+fieldTypes.website = {
-    validate: entity => {},
+    validate: field => {},
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeWebsite',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value
+};
+
+fieldTypes.longtext = {
+    validate: field => {},
     addColumn: (table, name) => table.text(name),
     indexed: false,
     grouped: false,
     enumerated: false,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeLongtext',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value
+};
+
+fieldTypes.gpg = {
+    validate: field => {},
+    addColumn: (table, name) => table.text(name),
+    indexed: false,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeGpg',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value
 };
 
 fieldTypes.json = {
-    validate: entity => {},
+    validate: field => {},
     addColumn: (table, name) => table.json(name),
     indexed: false,
     grouped: false,
     enumerated: false,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeJson',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => value
 };
 
 fieldTypes.number = {
-    validate: entity => {},
+    validate: field => {},
     addColumn: (table, name) => table.integer(name),
     indexed: true,
     grouped: false,
     enumerated: false,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeNumber',
+    forHbs: (field, value) => value,
+    parsePostValue: (field, value) => Number(value)
 };
 
 fieldTypes['checkbox-grouped'] = {
-    validate: entity => {},
+    validate: field => {},
     indexed: true,
     grouped: true,
     enumerated: false,
-    cardinality: Cardinality.MULTIPLE
+    cardinality: Cardinality.MULTIPLE,
+    getHbsType: field => 'typeCheckboxGrouped'
 };
 
-fieldTypes['radio-grouped'] = fieldTypes['dropdown-grouped'] = {
+fieldTypes['radio-grouped'] = {
-    validate: entity => {},
+    validate: field => {},
     indexed: true,
     grouped: true,
     enumerated: false,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeRadioGrouped'
 };
 
-fieldTypes['radio-enum'] = fieldTypes['dropdown-enum'] = {
+fieldTypes['dropdown-grouped'] = {
-    validate: entity => {
+    validate: field => {},
-        enforce(entity.settings.options, 'Options missing in settings');
+    indexed: true,
-        enforce(entity.default_value === null || entity.settings.options.find(x => x.key === entity.default_value), 'Default value not present in options');
+    grouped: true,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDropdownGrouped'
+};
+
+fieldTypes['radio-enum'] = {
+    validate: field => {
+        enforce(field.settings.options, 'Options missing in settings');
+        enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
     },
     addColumn: (table, name) => table.string(name),
     indexed: true,
     grouped: false,
     enumerated: true,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeRadioEnum'
+};
+
+fieldTypes['dropdown-enum'] = {
+    validate: field => {
+        enforce(field.settings.options, 'Options missing in settings');
+        enforce(field.default_value === null || field.settings.options.find(x => x.key === field.default_value), 'Default value not present in options');
+    },
+    addColumn: (table, name) => table.string(name),
+    indexed: true,
+    grouped: false,
+    enumerated: true,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDropdownEnum'
 };
 
 fieldTypes.option = {
-    validate: entity => {},
+    validate: field => {},
     addColumn: (table, name) => table.boolean(name),
     indexed: true,
     grouped: false,
@@ -95,15 +160,32 @@ fieldTypes.option = {
     cardinality: Cardinality.SINGLE
 };
 
-fieldTypes['date'] = fieldTypes['birthday'] = {
+fieldTypes['date'] = {
-    validate: entity => {
+    validate: field => {
-        enforce(['eur', 'us'].includes(entity.settings.dateFormat), 'Date format incorrect');
+        enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
     },
     addColumn: (table, name) => table.dateTime(name),
     indexed: true,
     grouped: false,
     enumerated: false,
-    cardinality: Cardinality.SINGLE
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeDate' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
+    forHbs: (field, value) => formatDate(field.settings.dateFormat, value),
+    parsePostValue: (field, value) => parseDate(field.settings.dateFormat, value)
+};
+
+fieldTypes['birthday'] = {
+    validate: field => {
+        enforce(['eur', 'us'].includes(field.settings.dateFormat), 'Date format incorrect');
+    },
+    addColumn: (table, name) => table.dateTime(name),
+    indexed: true,
+    grouped: false,
+    enumerated: false,
+    cardinality: Cardinality.SINGLE,
+    getHbsType: field => 'typeBirthday' + field.settings.dateFormat.charAt(0).toUpperCase() + field.settings.dateFormat.slice(1),
+    forHbs: (field, value) => formatBirthday(field.settings.dateFormat, value),
+    parsePostValue: (field, value) => parseBirthday(field.settings.dateFormat, value)
 };
 
 const groupedTypes = Object.keys(fieldTypes).filter(key => fieldTypes[key].grouped);
@@ -153,7 +235,7 @@ async function listTx(tx, listId) {
 
 async function list(context, listId) {
     return await knex.transaction(async tx => {
-        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageFields', 'manageSegments']);
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, ['viewSubscriptions', 'manageFields', 'manageSegments', 'manageSubscriptions']);
         return await listTx(tx, listId);
     });
 }
@@ -474,32 +556,127 @@ async function removeAllByListIdTx(tx, context, listId) {
     }
 }
 
-async function getRow(context, listId, subscription) {
+// Returns an array that can be used for rendering by Handlebars
+async function forHbs(context, listId, subscription) { // assumes grouped subscription
     const customFields = [{
         name: 'Email Address',
         column: 'email',
+        key: 'EMAIL',
         typeSubscriptionEmail: true,
         value: subscription ? subscription.email : '',
         order_subscribe: -1,
         order_manage: -1
     }];
 
-    const flds = await list(context, listId);
+    const flds = await listGrouped(context, listId);
 
     for (const fld of flds) {
-        if (fld.column) {
+        const type = fieldTypes[fld.type];
-            customFields.push({
+        const fldKey = getFieldKey(fld);
+
+        const entry = {
             name: fld.name,
-                column: fld.column,
+            key: fld.key,
-                ['type' + fld.type.replace(/(?:^|-)([a-z])/g, (m, c) => c.toUpperCase())]: true,
+            [type.getHbsType(fld)]: true,
-                value: subscription ? subscription[fld.column] : ''
+            order_subscribe: fld.order_subscribe,
+            order_manage: fld.order_manage,
+        };
+
+        if (!type.grouped && !type.enumerated) {
+            entry.value = subscription ? type.forHbs(fld, subscription[fldKey]) : '';
+
+        } else if (type.grouped) {
+            const options = [];
+            const value = subscription ? subscription[fldKey] : (type.cardinality === Cardinality.SINGLE ? null : []);
+
+            for (const optCol in fld.groupedOptions) {
+                const opt = fld.groupedOptions[optCol];
+
+                let isEnabled;
+                if (type.cardinality === Cardinality.SINGLE) {
+                    isEnabled = value === opt.column;
+                } else {
+                    isEnabled = value.includes(opt.column);
+                }
+
+                options.push({
+                    key: opt.key,
+                    name: opt.name,
+                    value: isEnabled
                 });
             }
+
+            entry.options = options;
+
+        } else if (type.enumerated) {
+            const options = [];
+            const value = subscription ? subscription[fldKey] : null;
+
+            for (const opt of fld.settings.options) {
+                options.push({
+                    key: opt.key,
+                    name: opt.label,
+                    value: value === opt.key
+                });
+            }
+
+            entry.options = options;
+
+        }
+
+        customFields.push(entry);
     }
 
     return customFields;
 }
 
+async function fromPost(context, listId, data) { // assumes grouped subscription
+
+    const flds = await listGrouped(context, listId);
+
+    const subscription = {};
+
+    for (const fld of flds) {
+        const type = fieldTypes[fld.type];
+        const fldKey = getFieldKey(fld);
+
+        let value = null;
+
+        if (!type.grouped && !type.enumerated) {
+            value = type.parsePostValue(fld, cleanupFromPost(data[fld.key]));
+
+        } else if (type.grouped) {
+            if (type.cardinality === Cardinality.SINGLE) {
+                for (const optCol in fld.groupedOptions) {
+                    const opt = fld.groupedOptions[optCol];
+
+                    if (data[fld.key] === opt.key) {
+                        value = opt.column
+                    }
+                }
+            } else {
+                value = [];
+
+                for (const optCol in fld.groupedOptions) {
+                    const opt = fld.groupedOptions[optCol];
+
+                    if (data[opt.key]) {
+                        value.push(opt.column);
+                    }
+                }
+            }
+
+        } else if (type.enumerated) {
+            value = data[fld.key];
+        }
+
+        subscription[fldKey] = value;
+    }
+
+    return subscription;
+}
+
+
 // This is to handle circular dependency with segments.js
 Object.assign(module.exports, {
     Cardinality,
@@ -518,5 +695,6 @@ Object.assign(module.exports, {
     remove,
     removeAllByListIdTx,
     serverValidate,
-    getRow
+    forHbs,
+    fromPost
 });

models/shares.js

@@ -8,6 +8,9 @@ const dtHelpers = require('../lib/dt-helpers');
 const permissions = require('../lib/permissions');
 const interoperableErrors = require('../shared/interoperable-errors');
 
+// TODO: This would really benefit from some permission cache connected to rebuildPermissions
+// A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions
+
 
 async function listByEntityDTAjax(context, entityTypeId, entityId, params) {
     return await knex.transaction(async (tx) => {

models/subscriptions.js

@@ -31,17 +31,7 @@ function getOptionsMap(groupedField) {
     return result;
 }
 
-fieldTypes.text = fieldTypes.website = fieldTypes.longtext = fieldTypes.gpg = {
+fieldTypes.text = fieldTypes.website = fieldTypes.longtext = fieldTypes.gpg = fieldTypes.number = fieldTypes.json = {
-    afterJSON: (groupedField, entity) => {},
-    listRender: (groupedField, value) => value
-};
-
-fieldTypes.number = {
-    afterJSON: (groupedField, entity) => {},
-    listRender: (groupedField, value) => Number(value)
-};
-
-fieldTypes.json = {
     afterJSON: (groupedField, entity) => {},
     listRender: (groupedField, value) => value
 };
@@ -207,26 +197,16 @@ async function hashByList(listId, entity) {
 }
 
 
-async function _getStatusBy(context, listId, key, value) {
-    return await knex.transaction(async tx => {
-        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
-
-        const entity = await tx(getSubscriptionTableName(listId)).where(key, value).select(['status']).first();
-
-        if (!entity) {
-            throw new interoperableErrors.NotFoundError();
-        }
-
-        return entity.status;
-    });
-}
-
 async function _getBy(context, listId, key, value, grouped) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
 
         const entity = await tx(getSubscriptionTableName(listId)).where(key, value).first();
 
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError('Subscription not found in this list');
+        }
+
         const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
 
         if (grouped) {
@@ -355,19 +335,36 @@ async function listDTAjax(context, listId, segmentId, params) {
     });
 }
 
-async function list(context, listId) {
+async function list(context, listId, grouped = true, offset, limit) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions');
 
-        const entities = await tx(getSubscriptionTableName(listId));
+        const count = await tx(getSubscriptionTableName(listId)).count('* as count').first().count;
 
+        const entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc');
+
+        if (Number.isInteger(offset)) {
+            entitiesQry.offset(offset);
+        }
+
+        if (Number.isInteger(limit)) {
+            entitiesQry.limit(limit);
+        }
+
+        const entities = await entitiesQry;
+
+        if (grouped) {
             const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
 
             for (const entity of entities) {
                 groupSubscription(groupedFieldsMap, entity);
             }
+        }
 
-        return entities;
+        return {
+            subscriptions: entities,
+            total: count
+        };
     });
 }
 
@@ -395,7 +392,6 @@ async function serverValidate(context, listId, data) {
 
 async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) {
     enforce(entity.email, 'Email must be set');
-    enforce(entity.status > 0 && entity.status < SubscriptionStatus.MAX, 'Subscription status is invalid');
 
     const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('email', entity.email);
 
@@ -405,14 +401,16 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
     const existingWithKey = await existingWithKeyQuery.first();
     if (existingWithKey) {
         if (meta && meta.replaceOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED) {
-            meta.updateNeeded = true;
+            meta.update = true;
             meta.existing = existingWithKey;
         } else {
             throw new interoperableErrors.DuplicitEmailError();
         }
     }
 
+    if ((isCreate && !(meta && meta.update)) || 'status' in entity) {
         enforce(entity.status >= 0 && entity.status < SubscriptionStatus.MAX, 'Invalid status');
+    }
 
     for (const key in groupedFieldsMap) {
         const fld = groupedFieldsMap[key];
@@ -422,13 +420,15 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
 }
 
 async function _update(tx, listId, existing, filteredEntity) {
+    if ('status' in filteredEntity) {
         if (existing.status !== filteredEntity.status) {
             filteredEntity.status_change = new Date();
         }
+    }
 
     await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
 
-
+    if ('status' in filteredEntity) {
         let countIncrement = 0;
         if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
             countIncrement = -1;
@@ -440,6 +440,7 @@ async function _update(tx, listId, existing, filteredEntity) {
             await tx('lists').where('id', listId).increment('subscribers', countIncrement);
         }
     }
+}
 
 async function _create(tx, listId, filteredEntity) {
     const ids = await tx(getSubscriptionTableName(listId)).insert(filteredEntity);
@@ -452,6 +453,11 @@ async function _create(tx, listId, filteredEntity) {
     return id;
 }
 
+/*
+    Adds a new subscription. Returns error if a subscription with the same email address is already present and is not unsubscribed.
+    If it is unsubscribed, the existing subscription is changed based on the provided data.
+    If meta.partial is true, it updates even an active subscription.
+ */
 async function create(context, listId, entity, meta /* meta is provided when called from /confirm/subscribe/:cid */) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
@@ -470,7 +476,7 @@ async function create(context, listId, entity, meta /* meta is provided when cal
         filteredEntity.opt_in_country = meta && meta.country;
         filteredEntity.imported = meta && !!meta.imported;
 
-        if (meta && meta.updateNeeded) {
+        if (meta && meta.update) {
             await _update(tx, listId, meta.existing, filteredEntity);
             meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
             return meta.existing.id;
@@ -515,10 +521,9 @@ async function updateWithConsistencyCheck(context, listId, entity) {
     });
 }
 
-async function removeTx(tx, context, listId, id) {
+async function _removeAndGetTx(tx, context, listId, existing) {
     await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
 
-    const existing = await tx(getSubscriptionTableName(listId)).where('id', id).first();
     if (!existing) {
         throw new interoperableErrors.NotFoundError();
     }
@@ -532,10 +537,19 @@ async function removeTx(tx, context, listId, id) {
 
 async function remove(context, listId, id) {
     await knex.transaction(async tx => {
-        await removeTx(tx, context, listId, id);
+        const existing = await tx(getSubscriptionTableName(listId)).where('id', id).first();
+        await _removeAndGetTx(tx, context, listId, existing);
     });
 }
 
+async function removeByEmailAndGet(context, listId, email) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first();
+        return await _removeAndGetTx(tx, context, listId, existing);
+    });
+}
+
+
 async function _unsubscribeAndGetTx(tx, context, listId, existingSubscription, campaignCid) {
     await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
 
@@ -585,6 +599,13 @@ async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaign
     });
 }
 
+async function unsubscribeByEmailAndGet(context, listId, email) {
+    return await knex.transaction(async tx => {
+        const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first();
+        return _unsubscribeAndGetTx(tx, context, listId, existing);
+    });
+}
+
 async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
@@ -594,39 +615,60 @@ async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
             throw new interoperableErrors.NotFoundError();
         }
 
+        if (existing.email !== emailNew) {
+            await tx(getSubscriptionTableName(listId)).where('email', emailNew).del();
+
             await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
                 email: emailNew
             });
 
             existing.email = emailNew;
+        }
+
         return existing;
     });
 }
 
-async function updateManagedUngrouped(context, listId, entity) {
+async function updateManaged(context, listId, cid, entity) {
     await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
 
-        const existing = await tx(getSubscriptionTableName(listId)).where('cid', entity.cid).first();
+        const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
-        if (!existing || existing.status !== SubscriptionStatus.SUBSCRIBED) {
-            throw new interoperableErrors.NotFoundError();
-        }
-
-        const flds = await fields.listTx(tx, listId);
 
         const update = {};
+        for (const key in groupedFieldsMap) {
+            const fld = groupedFieldsMap[key];
 
-        for (const fld of flds) {
             if (fld.order_manage) {
-                if (!fld.group) { // fieldTypes is primarily meant only for groupedFields, so we don't try it for fields that would be grouped (i.e. option), because there is nothing to be done for them anyway
+                update[key] = entity[key];
-                    fieldTypes[fld.type].afterJSON(fld, entity);
             }
 
-                update[fld.column] = entity[fld.column];
+            fieldTypes[fld.type].afterJSON(fld, update);
+        }
+
+        ungroupSubscription(groupedFieldsMap, update);
+
+        await tx(getSubscriptionTableName(listId)).where('cid', cid).update(update);
+    });
+}
+
+
+async function getListsWithEmail(context, email) {
+    // FIXME - this methods is rather suboptimal if there are many lists. It quite needs permission caching in shares.js
+
+    return await knex.transaction(async tx => {
+        const lists = await tx('lists').select(['id', 'name']);
+        const result = [];
+
+        for (const list of lists) {
+            await shares.enforceEntityPermissionTx(tx, context, 'list', list.id, 'viewSubscriptions');
+            const entity = await tx(getSubscriptionTableName(list.id)).where('email', email).first();
+            if (entity) {
+                result.push(list);
             }
         }
 
-        await tx(getSubscriptionTableName(listId)).where('cid', entity.cid).update(update);
+        return result;
     });
 }
 
@@ -641,8 +683,11 @@ module.exports = {
     create,
     updateWithConsistencyCheck,
     remove,
+    removeByEmailAndGet,
     unsubscribeByCidAndGet,
     unsubscribeByIdAndGet,
+    unsubscribeByEmailAndGet,
     updateAddressAndGet,
-    updateManagedUngrouped
+    updateManaged,
+    getListsWithEmail
 };

routes/api.js

@@ -1,124 +1,82 @@
 'use strict';
 
-let lists = require('../lib/models/lists');
+const lists = require('../models/lists');
-let fields = require('../lib/models/fields');
+const tools = require('../lib/tools-async');
-let blacklist = require('../models/blacklist');
+const blacklist = require('../models/blacklist');
-let subscriptions = require('../lib/models/subscriptions');
+const fields = require('../models/fields');
-let confirmations = require('../lib/models/confirmations');
+const { SubscriptionStatus } = require('../shared/lists');
-let tools = require('../lib/tools');
+const subscriptions = require('../models/subscriptions');
-let log = require('npmlog');
+const confirmations = require('../models/confirmations');
+const log = require('npmlog');
 const router = require('../lib/router-async').create();
-let mailHelpers = require('../lib/subscription-mail-helpers');
+const mailHelpers = require('../lib/subscription-mail-helpers');
 const interoperableErrors = require('../shared/interoperable-errors');
+const contextHelpers = require('../lib/context-helpers');
+const shares = require('../models/shares');
+const slugify = require('slugify');
 
-router.post('/subscribe/:listId', (req, res) => {
+class APIError extends Error {
-    let input = {};
+    constructor(msg, status) {
+        super(msg);
+        this.status = status;
+    }
+}
+
+
+router.postAsync('/subscribe/:listId', async (req, res) => {
+    const listId = req.params.listId;
+    
+    const input = {};
     Object.keys(req.body).forEach(key => {
         input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
     });
-    lists.getByCid(req.params.listId, (err, list) => {
+
-        if (err) {
-            log.error('API', err);
-            res.status(500);
-            return res.json({
-                error: err.message || err,
-                data: []
-            });
-        }
-        if (!list) {
-            res.status(404);
-            return res.json({
-                error: 'Selected listId not found',
-                data: []
-            });
-        }
     if (!input.EMAIL) {
-            res.status(400);
+        throw new APIError('Missing EMAIL', 400);
-            return res.json({
-                error: 'Missing EMAIL',
-                data: []
-            });
-        }
-        tools.validateEmail(input.EMAIL, false, err => {
-            if (err) {
-                log.error('API', err);
-                res.status(400);
-                return res.json({
-                    error: err.message || err,
-                    data: []
-                });
     }
 
-            let subscription = {
+    const emailErr = await tools.validateEmail(input.EMAIL, false);
+    if (emailErr) {
+        const errMsg = tools.validateEmailGetMessage(emailErr, email);
+        log.error('API', errMsg);
+        throw new APIError(errMsg, 400);
+    }
+
+    const subscription = {
         email: input.EMAIL
     };
 
-            if (input.FIRST_NAME) {
-                subscription.first_name = (input.FIRST_NAME || '').toString().trim();
-            }
-
-            if (input.LAST_NAME) {
-                subscription.last_name = (input.LAST_NAME || '').toString().trim();
-            }
-
     if (input.TIMEZONE) {
         subscription.tz = (input.TIMEZONE || '').toString().trim();
     }
 
-            fields.list(list.id, (err, fieldList) => {
+    const fieldList = await fields.fromPost(req.context, listId);
-                if (err && !fieldList) {
-                    fieldList = [];
-                }
 
-                fieldList.forEach(field => {
+    for (const field of fieldList) {
-                    if (input.hasOwnProperty(field.key) && field.column) {
+        if (field.key in input && field.column) {
+            if (field.type === 'option') {
+                subscription[field.column] = ['false', 'no', '0', ''].indexOf((input[field.key] || '').toString().trim().toLowerCase()) >= 0 ? '' : '1';
+            } else {
                 subscription[field.column] = input[field.key];
-                    } else if (field.options) {
-                        for (let i = 0, len = field.options.length; i < len; i++) {
-                            if (input.hasOwnProperty(field.options[i].key) && field.options[i].column) {
-                                let value = input[field.options[i].key];
-                                if (field.options[i].type === 'option') {
-                                    value = ['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0 ? '' : '1';
-                                }
-                                subscription[field.options[i].column] = value;
             }
         }
     }
-                });
-
-                let meta = {
-                    partial: true
-                };
 
     if (/^(yes|true|1)$/i.test(input.FORCE_SUBSCRIBE)) {
-                    meta.status = 1;
+        subscription.status = SubscriptionStatus.SUBSCRIBED;
     }
 
     if (/^(yes|true|1)$/i.test(input.REQUIRE_CONFIRMATION)) {
+        const list = await lists.getByCid(contextHelpers.getAdminContext(), listId);
+        await shares.enforceEntityPermission(req.context, 'list', listId, 'manageSubscriptions');
+
         const data = {
-                        email: subscription.email,
+            email,
             subscriptionData: subscription
         };
 
-                    confirmations.addConfirmation(list.id, 'subscribe', req.ip, data, (err, confirmCid) => {
+        const confirmCid = await confirmations.addConfirmation(listId, 'subscribe', req.ip, data);
-                        if (err) {
+        await mailHelpers.sendConfirmSubscription(list, input.EMAIL, confirmCid, subscription);
-                            log.error('API', err);
-                            res.status(500);
-                            return res.json({
-                                error: err.message || err,
-                                data: []
-                            });
-                        }
-
-                        mailHelpers.sendConfirmSubscription(list, input.EMAIL, confirmCid, subscription, (err) => {
-                            if (err) {
-                                log.error('API', err);
-                                res.status(500);
-                                return res.json({
-                                    error: err.message || err,
-                                    data: []
-                                });
-                            }
 
         res.status(200);
         res.json({
@@ -126,84 +84,32 @@ router.post('/subscribe/:listId', (req, res) => {
                 id: confirmCid
             }
         });
-                        });
-                    });
     } else {
-                    subscriptions.insert(list.id, meta, subscription, (err, response) => {
+        const meta = {};
-                        if (err) {
+        await subscriptions.create(req.context, listId, subscription, meta);
-                            log.error('API', err);
+
-                            res.status(500);
-                            return res.json({
-                                error: err.message || err,
-                                data: []
-                            });
-                        }
         res.status(200);
         res.json({
             data: {
-                                id: response.cid
+                id: meta.cid
             }
         });
-                    });
     }
 });
-        });
-    });
-});
 
-router.post('/unsubscribe/:listId', (req, res) => {
+
-    let input = {};
+router.postAsync('/unsubscribe/:listId', async (req, res) => {
+    const input = {};
     Object.keys(req.body).forEach(key => {
         input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
     });
-    lists.getByCid(req.params.listId, (err, list) => {
+
-        if (err) {
-            res.status(500);
-            return res.json({
-                error: err.message || err,
-                data: []
-            });
-        }
-        if (!list) {
-            res.status(404);
-            return res.json({
-                error: 'Selected listId not found',
-                data: []
-            });
-        }
     if (!input.EMAIL) {
-            res.status(400);
+        throw new APIError('Missing EMAIL', 400);
-            return res.json({
-                error: 'Missing EMAIL',
-                data: []
-            });
     }
 
-        subscriptions.getByEmail(list.id, input.EMAIL, (err, subscription) => {
+    const subscription = await subscriptions.unsubscribeByEmailAndGet(req.context, req.params.listId, input.EMAIL);
-            if (err) {
-                res.status(500);
-                return res.json({
-                    error: err.message || err,
-                    data: []
-                });
-            }
 
-            if (!subscription) {
-                res.status(404);
-                return res.json({
-                    error: 'Subscription with given email not found',
-                    data: []
-                });
-            }
-
-            subscriptions.changeStatus(list.id, subscription.id, false, subscriptions.Status.UNSUBSCRIBED, (err, found) => {
-                if (err) {
-                    res.status(500);
-                    return res.json({
-                        error: err.message || err,
-                        data: []
-                    });
-                }
     res.status(200);
     res.json({
         data: {
@@ -212,67 +118,20 @@ router.post('/unsubscribe/:listId', (req, res) => {
         }
     });
 });
-        });
-    });
-});
 
-router.post('/delete/:listId', (req, res) => {
+
-    let input = {};
+router.postAsync('/delete/:listId', async (req, res) => {
+    const input = {};
     Object.keys(req.body).forEach(key => {
         input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
     });
-    lists.getByCid(req.params.listId, (err, list) => {
+
-        if (err) {
-            res.status(500);
-            return res.json({
-                error: err.message || err,
-                data: []
-            });
-        }
-        if (!list) {
-            res.status(404);
-            return res.json({
-                error: 'Selected listId not found',
-                data: []
-            });
-        }
     if (!input.EMAIL) {
-            res.status(400);
+        throw new APIError('Missing EMAIL', 400);
-            return res.json({
-                error: 'Missing EMAIL',
-                data: []
-            });
-        }
-        subscriptions.getByEmail(list.id, input.EMAIL, (err, subscription) => {
-            if (err) {
-                res.status(500);
-                return res.json({
-                    error: err.message || err,
-                    data: []
-                });
-            }
-            if (!subscription) {
-                res.status(404);
-                return res.json({
-                    error: 'Subscription not found',
-                    data: []
-                });
-            }
-            subscriptions.delete(list.id, subscription.cid, (err, subscription) => {
-                if (err) {
-                    res.status(500);
-                    return res.json({
-                        error: err.message || err,
-                        data: []
-                    });
-                }
-                if (!subscription) {
-                    res.status(404);
-                    return res.json({
-                        error: 'Subscription not found',
-                        data: []
-                    });
     }
+
+    const subscription = await subscriptions.removeByEmailAndGet(req.context, req.params.listId, input.EMAIL);
+
     res.status(200);
     res.json({
         data: {
@@ -281,59 +140,98 @@ router.post('/delete/:listId', (req, res) => {
         }
     });
 });
-        });
+
+
+router.getAsync('/subscriptions/:listId', async (req, res) => {
+    const start = parseInt(req.query.start || 0, 10);
+    const limit = parseInt(req.query.limit || 10000, 10);
+
+    const { subscriptions, total } = await subscriptions.list(req.params.listId, false, start, limit);
+
+    res.status(200);
+    res.json({
+        data: {
+            total: total,
+            start: start,
+            limit: limit,
+            subscriptions
+        }
     });
 });
 
-router.post('/field/:listId', (req, res) => {
+
-    let input = {};
+router.getAsync('/lists/:email', async (req, res) => {
+    const lists = await subscriptions.getListsWithEmail(req.context, req.params.email);
+
+    res.status(200);
+    res.json({
+        data: lists
+    });
+});
+
+
+router.postAsync('/field/:listId', async (req, res) => {
+    const input = {};
     Object.keys(req.body).forEach(key => {
         input[(key || '').toString().trim().toUpperCase()] = (req.body[key] || '').toString().trim();
     });
-    lists.getByCid(req.params.listId, (err, list) => {
+
-        if (err) {
+    const key = (input.NAME || '').toString().trim() || slugify('merge ' + name, '_').toUpperCase();
-            log.error('API', err);
+    const visible = ['false', 'no', '0', ''].indexOf((input.VISIBLE || '').toString().toLowerCase().trim()) < 0;
-            res.status(500);
+
-            return res.json({
+    const groupTemplate = (input.GROUP_TEMPLATE || '').toString().toLowerCase().trim();
-                error: err.message || err,
+
-                data: []
+    let type = (input.TYPE || '').toString().toLowerCase().trim();
-            });
+    const settings = {};
-        }
+
-        if (!list) {
+    if (type === 'checkbox') {
-            res.status(404);
+        type = 'checkbox-grouped';
-            return res.json({
+        settings.groupTemplate = groupTemplate;
-                error: 'Selected listId not found',
+    } else if (type === 'dropdown') {
-                data: []
+        type = 'dropdown-grouped';
-            });
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'radio') {
+        type = 'radio-grouped';
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'json') {
+        settings.groupTemplate = groupTemplate;
+    } else if (type === 'date-us') {
+        type = 'date';
+        settings.dateFormat = 'us';
+    } else if (type === 'date-eur') {
+        type = 'date';
+        settings.dateFormat = 'eur';
+    } else if (type === 'birthday-us') {
+        type = 'birthday';
+        settings.birthdayFormat = 'us';
+    } else if (type === 'birthday-eur') {
+        type = 'birthday';
+        settings.birthdayFormat = 'eur';
     }
 
-        let field = {
+    const field = {
         name: (input.NAME || '').toString().trim(),
-            defaultValue: (input.DEFAULT || '').toString().trim() || null,
+        key,
-            type: (input.TYPE || '').toString().toLowerCase().trim(),
+        default_value: (input.DEFAULT || '').toString().trim() || null,
+        type,
+        settings,
         group: Number(input.GROUP) || null,
-            groupTemplate: (input.GROUP_TEMPLATE || '').toString().toLowerCase().trim(),
+        orderListBefore: visible ? 'end' : 'none',
-            visible: ['false', 'no', '0', ''].indexOf((input.VISIBLE || '').toString().toLowerCase().trim()) < 0
+        orderSubscribeBefore: visible ? 'end' : 'none',
+        orderManageBefore: visible ? 'end' : 'none'
     };
 
-        fields.create(list.id, field, (err, id, tag) => {
+    const id = await fields.create(req.context, req.params.listId, field);
-            if (err) {
+
-                res.status(500);
-                return res.json({
-                    error: err.message || err,
-                    data: []
-                });
-            }
     res.status(200);
     res.json({
         data: {
             id,
-                    tag
+            tag: key
         }
     });
 });
-    });
+
-});
 
 router.postAsync('/blacklist/add', async (req, res) => {
     let input = {};
@@ -351,6 +249,7 @@ router.postAsync('/blacklist/add', async (req, res) => {
     });
 });
 
+
 router.postAsync('/blacklist/delete', async (req, res) => {
     let input = {};
     Object.keys(req.body).forEach(key => {
@@ -367,6 +266,7 @@ router.postAsync('/blacklist/delete', async (req, res) => {
     });
 });
 
+
 router.getAsync('/blacklist/get', async (req, res) => {
     let start = parseInt(req.query.start || 0, 10);
     let limit = parseInt(req.query.limit || 10000, 10);
@@ -384,4 +284,5 @@ router.getAsync('/blacklist/get', async (req, res) => {
     });
 });
 
+
 module.exports = router;

routes/subscription.js

@@ -37,6 +37,7 @@ const objectHash = require('object-hash');
 const bluebird = require('bluebird');
 const fsReadFile = bluebird.promisify(require('fs').readFile);
 
+const { cleanupFromPost } = require('../lib/helpers');
 
 const originWhitelist = config.cors && config.cors.origins || [];
 
@@ -158,8 +159,6 @@ router.getAsync('/confirm/subscribe/:cid', async (req, res) => {
         }
     }
 
-    console.log(subscription);
-
     const list = await lists.getById(contextHelpers.getAdminContext(), confirmation.list);
     subscription.cid = meta.cid;
     await mailHelpers.sendSubscriptionConfirmed(list, subscription.email, subscription);
@@ -195,31 +194,15 @@ router.getAsync('/confirm/unsubscribe/:cid', async (req, res) => {
 });
 
 
-router.getAsync('/:cid', passport.csrfProtection, async (req, res) => {
+async function _renderSubscribe(req, res, list, subscription) {
-    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+    const data = {};
-
+    data.email = subscription && subscription.email;
-    if (!list.public_subscribe) {
-        shares.throwPermissionDenied();
-    }
-
-    const ucid = req.query.cid;
-
-    const data = req.query;
     data.layout = 'subscription/layout';
     data.title = list.name;
     data.cid = list.cid;
     data.csrfToken = req.csrfToken();
 
-    let subscription;
+    data.customFields = await fields.forHbs(contextHelpers.getAdminContext(), list.id, subscription);
-    if (ucid) {
-        subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, ucid, false);
-
-        if (subscription) {
-            data.email = subscription.email;
-        }
-    }
-
-    data.customFields = await fields.getRow(contextHelpers.getAdminContext(), list.id, subscription);
     data.useEditor = true;
 
     const configItems = await settings.get(['pgpPrivateKey', 'defaultAddress', 'defaultPostaddress']);
@@ -241,68 +224,63 @@ router.getAsync('/:cid', passport.csrfProtection, async (req, res) => {
     data.flashMessages = await captureFlashMessages(res);
 
     const result = htmlRenderer(data);
+
     res.send(result);
-});
-
-
-router.options('/:cid/widget', cors(corsOptions));
-
-router.getAsync('/:cid/widget', cors(corsOptions), async (req, res) => {
-    req.needsAPIJSONResponse = true;
-
-    const cached = cache.get(req.path);
-    if (cached) {
-        return res.status(200).json(cached);
 }
 
+router.getAsync('/:cid', passport.csrfProtection, async (req, res) => {
     const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
 
-    const configItems = await settings.get(['serviceUrl', 'pgpPrivateKey']);
+    if (!list.public_subscribe) {
-
+        shares.throwPermissionDenied();
-    const data = {
-        title: list.name,
-        cid: list.cid,
-        serviceUrl: configItems.serviceUrl,
-        hasPubkey: !!configItems.pgpPrivateKey,
-        customFields: await fields.getRow(contextHelpers.getAdminContext(), list.id),
-        template: {},
-        layout: null,
-    };
-
-    await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-subscribe', data);
-
-    const renderAsync = bluebird.promisify(res.render);
-    const html = await renderAsync('subscription/widget-subscribe', data);
-
-    const response = {
-        data: {
-            title: data.title,
-            cid: data.cid,
-            html
     }
-    };
 
-    cache.put(req.path, response, 30000); // ms
+    const ucid = req.query.cid;
-    res.status(200).json(response);
+
+    let subscription;
+    if (ucid) {
+        try {
+            subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, ucid);
+
+            if (subscription.status === SubscriptionStatus.SUBSCRIBED) {
+                subscription = null;
+            }
+        } catch (err) {
+            if (err instanceof interoperableErrors.NotFoundError) {
+            } else {
+                throw err;
+            }
+        }
+    }
+
+    await _renderSubscribe(req, res, list, subscription);
 });
 
 
 router.options('/:cid/subscribe', cors(corsOptions));
 
 router.postAsync('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, async (req, res) => {
-    const email = (req.body.email || '').toString().trim();
-
     if (req.xhr) {
         req.needsAPIJSONResponse = true;
     }
 
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+
+    if (!list.public_subscribe) {
+        shares.throwPermissionDenied();
+    }
+
+    const subscriptionData = await fields.fromPost(contextHelpers.getAdminContext(), list.id, req.body);
+
+    const email = cleanupFromPost(req.body.EMAIL);
+
     if (!email) {
         if (req.xhr) {
             throw new Error('Email address not set');
         }
 
         req.flash('danger', _('Email address not set'));
-        return res.redirect('/subscription/' + encodeURIComponent(req.params.cid) + '?' + tools.queryParams(req.body));
+        return await _renderSubscribe(req, res, list, subscriptionData);
     }
 
     const emailErr = await tools.validateEmail(email);
@@ -314,7 +292,8 @@ router.postAsync('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, as
         }
 
         req.flash('danger', errMsg);
-        return res.redirect('/subscription/' + encodeURIComponent(req.params.cid) + '?' + tools.queryParams(req.body));
+        subscriptionData.email = email;
+        return await _renderSubscribe(req, res, list, subscriptionData);
     }
 
     // Check if the subscriber seems legit. This is a really simple check, the only requirement is that
@@ -326,23 +305,18 @@ router.postAsync('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, as
     let addressTest = !req.body.address;
     let testsPass = subTimeTest && addressTest;
 
-    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+    let existingSubscription;
-
+    try {
-    if (!list.public_subscribe) {
+        existingSubscription = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, email);
-        shares.throwPermissionDenied();
+    } catch (err) {
+        if (err instanceof interoperableErrors.NotFoundError) {
+        } else {
+            throw err;
+        }
     }
 
-    const subscriptionData = {};
+    if (existingSubscription && existingSubscription.status === SubscriptionStatus.SUBSCRIBED) {
-    Object.keys(req.body).forEach(key => {
+        await mailHelpers.sendAlreadySubscribed(list, email, existingSubscription);
-        if (key !== 'email' && key.charAt(0) !== '_') {
-            subscriptionData[key] = (req.body[key] || '').toString().trim();
-        }
-    });
-
-    const subscription = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, email, false);
-
-    if (subscription && subscription.status === SubscriptionStatus.SUBSCRIBED) {
-        await mailHelpers.sendAlreadySubscribed(list, email, subscription);
         res.redirect('/subscription/' + encodeURIComponent(req.params.cid) + '/confirm-subscription-notice');
 
     } else {
@@ -368,11 +342,56 @@ router.postAsync('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, as
     }
 });
 
+
+router.options('/:cid/widget', cors(corsOptions));
+
+router.getAsync('/:cid/widget', cors(corsOptions), async (req, res) => {
+    req.needsAPIJSONResponse = true;
+
+    const cached = cache.get(req.path);
+    if (cached) {
+        return res.status(200).json(cached);
+    }
+
+    const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
+
+    const configItems = await settings.get(['serviceUrl', 'pgpPrivateKey']);
+
+    const data = {
+        title: list.name,
+        cid: list.cid,
+        serviceUrl: configItems.serviceUrl,
+        hasPubkey: !!configItems.pgpPrivateKey,
+        customFields: await fields.forHbs(contextHelpers.getAdminContext(), list.id),
+        template: {},
+        layout: null,
+    };
+
+    await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-subscribe', data);
+
+    const renderAsync = bluebird.promisify(res.render);
+    const html = await renderAsync('subscription/widget-subscribe', data);
+
+    const response = {
+        data: {
+            title: data.title,
+            cid: data.cid,
+            html
+        }
+    };
+
+    cache.put(req.path, response, 30000); // ms
+    res.status(200).json(response);
+});
+
+
+
 router.getAsync('/:lcid/manage/:ucid', passport.csrfProtection, async (req, res) => {
     const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
-    const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid, false);
 
-    if (!subscription || subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+    const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid);
+
+    if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
         throw new interoperableErrors.NotFoundError('Subscription not found in this list');
     }
 
@@ -384,7 +403,7 @@ router.getAsync('/:lcid/manage/:ucid', passport.csrfProtection, async (req, res)
     data.csrfToken = req.csrfToken();
     data.layout = 'data/layout';
 
-    data.customFields = await fields.getRow(contextHelpers.getAdminContext(), list.id, subscription);
+    data.customFields = await fields.forHbs(contextHelpers.getAdminContext(), list.id, subscription);
 
     data.useEditor = true;
 
@@ -414,7 +433,8 @@ router.postAsync('/:lcid/manage', passport.parseForm, passport.csrfProtection, a
     const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
 
     try {
-        await subscriptions.updateManagedUngrouped(contextHelpers.getAdminContext(), list.id, req.body);
+        const subscriptionData = await fields.fromPost(contextHelpers.getAdminContext(), list.id, req.body);
+        await subscriptions.updateManaged(contextHelpers.getAdminContext(), list.id, req.body.cid, subscriptionData);
     } catch (err) {
         if (err instanceof interoperableErrors.NotFoundError) {
             throw new interoperableErrors.NotFoundError('Subscription not found in this list');
@@ -430,7 +450,7 @@ router.getAsync('/:lcid/manage-address/:ucid', passport.csrfProtection, async (r
     const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
     const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid, false);
 
-    if (!subscription || subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+    if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
         throw new interoperableErrors.NotFoundError('Subscription not found in this list');
     }
 
@@ -450,7 +470,7 @@ router.getAsync('/:lcid/manage-address/:ucid', passport.csrfProtection, async (r
         layout: 'subscription/layout.mjml.hbs'
     };
 
-    await injectCustomFormData(req.query.fid || list.default_form, 'data/web-manage-address', subscription);
+    await injectCustomFormData(req.query.fid || list.default_form, 'data/web-manage-address', data);
 
     const htmlRenderer = await getMjmlTemplate(data.template);
 
@@ -466,7 +486,7 @@ router.getAsync('/:lcid/manage-address/:ucid', passport.csrfProtection, async (r
 router.postAsync('/:lcid/manage-address', passport.parseForm, passport.csrfProtection, async (req, res) => {
     const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
 
-    const emailNew = (req.body['email-new'] || '').toString().trim();
+    const emailNew = cleanupFromPost(req.body['EMAIL_NEW']);
 
     const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.body.cid, false);
 
@@ -485,7 +505,15 @@ router.postAsync('/:lcid/manage-address', passport.parseForm, passport.csrfProte
             req.flash('danger', errMsg);
 
         } else {
-            const newSubscription = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, emailNew, false);
+            let newSubscription;
+            try {
+                newSubscription = await subscriptions.getByEmail(contextHelpers.getAdminContext(), list.id, emailNew, false);
+            } catch (err) {
+                if (err instanceof interoperableErrors.NotFoundError) {
+                } else {
+                    throw err;
+                }
+            }
 
             if (newSubscription && newSubscription.status === SubscriptionStatus.SUBSCRIBED) {
                 await mailHelpers.sendAlreadySubscribed(list, emailNew, subscription);
@@ -523,7 +551,7 @@ router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req,
 
         const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, req.params.ucid, false);
 
-        if (!subscription || subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+        if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
             throw new interoperableErrors.NotFoundError('Subscription not found in this list');
         }
 
@@ -562,7 +590,7 @@ router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req,
 router.postAsync('/:lcid/unsubscribe', passport.parseForm, passport.csrfProtection, async (req, res) => {
     const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
 
-    const campaignCid = (req.body.campaign || '').toString().trim() || false;
+    const campaignCid = cleanupFromPost(req.body.campaign);
 
     await handleUnsubscribe(list, req.body.ucid, false, campaignCid, req.ip, res);
 });
@@ -588,7 +616,7 @@ async function handleUnsubscribe(list, subscriptionCid, autoUnsubscribe, campaig
     } else {
         const subscription = await subscriptions.getByCid(contextHelpers.getAdminContext(), list.id, subscriptionCid, false);
 
-        if (!subscription || subscription.status !== SubscriptionStatus.SUBSCRIBED) {
+        if (subscription.status !== SubscriptionStatus.SUBSCRIBED) {
             throw new interoperableErrors.NotFoundError('Subscription not found in this list');
         }
 

test/e2e/index.js

@@ -8,7 +8,7 @@ const only = 'only';
 const skip = 'skip';
 
 let tests = [
-    'login',
+    //'login',
     'subscription'
 ];
 

test/e2e/page-objects/subscription.js

@@ -21,7 +21,7 @@ const fieldHelpers = list => ({
 
         for (const field of list.customFields) {
             if (field.key in subscription) {
-                await this.setValue(`[name="${field.column}"]`, subscription[field.key]);
+                await this.setValue(`[name="${field.key}"]`, subscription[field.key]);
             }
         }
     },
@@ -41,7 +41,7 @@ const fieldHelpers = list => ({
 
         for (const field of list.customFields) {
             if (field.key in subscription) {
-                expect(await this.getValue(`[name="${field.column}"]`)).to.equal(subscription[field.key]);
+                expect(await this.getValue(`[name="${field.key}"]`)).to.equal(subscription[field.key]);
             }
         }
     }
@@ -55,16 +55,29 @@ module.exports = list => ({
         textsToWaitFor: ['Subscribe to list'],
         elements: {
             form: `form[action="/subscription/${list.cid}/subscribe"]`,
-            emailInput: '#main-form input[name="email"]',
+            emailInput: '#main-form input[name="EMAIL"]',
-            firstNameInput: '#main-form input[name="first-name"]',
+            firstNameInput: '#main-form input[name="FIRST_NAME"]',
-            lastNameInput: '#main-form input[name="last-name"]',
+            lastNameInput: '#main-form input[name="LAST_NAME"]',
+            submitButton: 'a[href="#submit"]'
+        }
+    }, fieldHelpers(list)),
+
+    webSubscribeAfterPost: web({
+        url: `/subscription/${list.cid}/subscribe`,
+        elementsToWaitFor: ['form'],
+        textsToWaitFor: ['Subscribe to list'],
+        elements: {
+            form: `form[action="/subscription/${list.cid}/subscribe"]`,
+            emailInput: '#main-form input[name="EMAIL"]',
+            firstNameInput: '#main-form input[name="FIRST_NAME"]',
+            lastNameInput: '#main-form input[name="LAST_NAME"]',
             submitButton: 'a[href="#submit"]'
         }
     }, fieldHelpers(list)),
 
     webSubscribeNonPublic: web({
         url: `/subscription/${list.cid}`,
-        textsToWaitFor: ['The list does not allow public subscriptions'],
+        textsToWaitFor: ['Permission denied'],
     }),
 
     webConfirmSubscriptionNotice: web({
@@ -117,9 +130,9 @@ module.exports = list => ({
         textsToWaitFor: ['Update Your Preferences'],
         elements: {
             form: `form[action="/subscription/${list.cid}/manage"]`,
-            emailInput: '#main-form input[name="email"]',
+            emailInput: '#main-form input[name="EMAIL"]',
-            firstNameInput: '#main-form input[name="first-name"]',
+            firstNameInput: '#main-form input[name="FIRST_NAME"]',
-            lastNameInput: '#main-form input[name="last-name"]',
+            lastNameInput: '#main-form input[name="LAST_NAME"]',
             submitButton: 'a[href="#submit"]',
             manageAddressLink: `a[href^="/subscription/${list.cid}/manage-address/"]`
         },
@@ -134,8 +147,8 @@ module.exports = list => ({
         textsToWaitFor: ['Update Your Email Address'],
         elements: {
             form: `form[action="/subscription/${list.cid}/manage-address"]`,
-            emailInput: '#main-form input[name="email"]',
+            emailInput: '#main-form input[name="EMAIL"]',
-            emailNewInput: '#main-form input[name="email-new"]',
+            emailNewInput: '#main-form input[name="EMAIL_NEW"]',
             submitButton: 'a[href="#submit"]'
         }
     }),

test/e2e/tests/subscription.js

@@ -15,7 +15,7 @@ function getPage(listConf) {
 }
 
 function generateEmail() {
-    return 'keep.' + shortid.generate() + '@mailtrain.org';
+    return 'keep.' + shortid.generate() + '@gmail.com';
 }
 
 function generateCustomFieldValue(field) {
@@ -142,8 +142,8 @@ suite('Subscription use-cases', () => {
         });
 
         await step('System shows a flash notice that email is invalid.', async () => {
-            await page.webSubscribe.waitForFlash();
+            await page.webSubscribeAfterPost.waitForFlash();
-            expect(await page.webSubscribe.getFlash()).to.contain('Invalid email address');
+            expect(await page.webSubscribeAfterPost.getFlash()).to.contain('Invalid email address');
         });
     });
 

views/subscription/partials/subscription-custom-fields.hbs

@@ -2,74 +2,60 @@
 
     {{#if typeSubscriptionEmail}}
         <div class="form-group email">
-            <label for="email">{{#translate}}Email Address{{/translate}}</label>
+            <label for="EMAIL">{{#translate}}Email Address{{/translate}}</label>
             {{#if ../isManagePreferences}}
                 <div class="input-group">
-                    <input type="email" name="email" id="email" placeholder="" value="{{../email}}" readonly>
+                    <input type="email" name="EMAIL" id="email" placeholder="" value="{{../email}}" readonly>
                     <div class="input-group-addon"><a href="/subscription/{{../lcid}}/manage-address/{{../cid}}">{{#translate}}want to change it?{{/translate}}</a></div>
                 </div>
             {{else}}
-                <input type="email" name="email" id="email" placeholder="" value="{{../email}}" required>
+                <input type="email" name="EMAIL" id="email" placeholder="" value="{{../email}}" required>
             {{/if}}
         </div>
     {{/if}}
 
-    {{#if typeFirstName}}
-        <div class="form-group first-name">
-            <label for="first-name">{{#translate}}First Name{{/translate}}</label>
-            <input type="text" name="first-name" id="first-name" placeholder="" value="{{../firstName}}">
-        </div>
-    {{/if}}
-
-    {{#if typeLastName}}
-        <div class="form-group last-name">
-            <label for="last-name">{{#translate}}Last Name{{/translate}}</label>
-            <input type="text" name="last-name" id="last-name" placeholder="" value="{{../lastName}}">
-        </div>
-    {{/if}}
-
     {{#if typeText}}
-        <div class="form-group text {{column}}">
+        <div class="form-group text {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="text" name="{{column}}" value="{{value}}">
+            <input type="text" name="{{key}}" value="{{value}}">
         </div>
     {{/if}}
 
     {{#if typeNumber}}
-        <div class="form-group number {{column}}">
+        <div class="form-group number {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="number" name="{{column}}" value="{{value}}">
+            <input type="number" name="{{key}}" value="{{value}}">
         </div>
     {{/if}}
 
     {{#if typeWebsite}}
-        <div class="form-group url {{column}}">
+        <div class="form-group url {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="url" name="{{column}}" value="{{value}}">
+            <input type="url" name="{{key}}" value="{{value}}">
         </div>
     {{/if}}
 
     {{#if typeLongtext}}
-        <div class="form-group longtext {{column}}">
+        <div class="form-group longtext {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <textarea rows="3" name="{{column}}">{{value}}</textarea>
+            <textarea rows="3" name="{{key}}">{{value}}</textarea>
         </div>
     {{/if}}
 
     {{#if typeJson}}
-        <div class="form-group json {{column}}">
+        <div class="form-group json {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <textarea class="gpg-text" rows="3" name="{{column}}" placeholder="{&quot;data&quot;:&quot;value&quot;}">{{value}}</textarea>
+            <textarea class="gpg-text" rows="3" name="{{key}}" placeholder="{&quot;data&quot;:&quot;value&quot;}">{{value}}</textarea>
         </div>
     {{/if}}
 
     {{#if typeGpg}}
-        <div class="form-group gpg {{column}}">
+        <div class="form-group gpg {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
             {{#if ../hasPubkey}}
                 <button class="btn-download-pubkey" type="submit" form="download-pubkey">{{#translate}}Download signature verification key{{/translate}}</button>
             {{/if}}
-            <textarea class="form-control gpg-text" rows="4" name="{{column}}" placeholder="{{#translate}}Begins with{{/translate}} &#39;-----BEGIN PGP PUBLIC KEY BLOCK-----&#39;">{{value}}</textarea>
+            <textarea class="form-control gpg-text" rows="4" name="{{key}}" placeholder="{{#translate}}Begins with{{/translate}} &#39;-----BEGIN PGP PUBLIC KEY BLOCK-----&#39;">{{value}}</textarea>
             <span class="help-block">
                 {{#translate}}Insert your GPG public key here to encrypt messages sent to your address{{/translate}} <em>({{#translate}}optional{{/translate}})</em>
             </span>
@@ -77,34 +63,34 @@
     {{/if}}
 
     {{#if typeDateUs}}
-        <div class="form-group date fm-date-us {{column}}">
+        <div class="form-group date fm-date-us {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="text" name="{{column}}" placeholder="MM/DD/YYYY" value="{{value}}">
+            <input type="text" name="{{key}}" placeholder="MM/DD/YYYY" value="{{value}}">
         </div>
     {{/if}}
 
     {{#if typeDateEur}}
-        <div class="form-group date fm-date-eur {{column}}">
+        <div class="form-group date fm-date-eur {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="text" name="{{column}}" placeholder="DD/MM/YYYY" value="{{value}}">
+            <input type="text" name="{{key}}" placeholder="DD/MM/YYYY" value="{{value}}">
         </div>
     {{/if}}
 
     {{#if typeBirthdayUs}}
-        <div class="form-group date fm-birthday-us {{column}}">
+        <div class="form-group date fm-birthday-us {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="text" name="{{column}}" placeholder="MM/DD" value="{{value}}">
+            <input type="text" name="{{key}}" placeholder="MM/DD" value="{{value}}">
         </div>
     {{/if}}
 
     {{#if typeBirthdayEur}}
-        <div class="form-group date fm-birthday-eur {{column}}">
+        <div class="form-group date fm-birthday-eur {{key}}">
-            <label for="{{column}}">{{name}}</label>
+            <label for="{{key}}">{{name}}</label>
-            <input type="text" name="{{column}}" placeholder="DD/MM" value="{{value}}">
+            <input type="text" name="{{key}}" placeholder="DD/MM" value="{{value}}">
         </div>
     {{/if}}
 
-    {{#if typeDropdown}}
+    {{#if typeDropdownGrouped }}
         <div class="form-group dropdown {{key}}">
             <label for="{{key}}">{{name}}</label>
             <select name="{{key}}" class="form-control">
@@ -112,29 +98,54 @@
                     –– {{#translate}}Select{{/translate}} ––
                 </option>
                 {{#each options}}
-                    <option value="{{column}}" {{#if value}} selected {{/if}}>{{name}}</option>
+                    <option value="{{key}}" {{#if value}} selected {{/if}}>{{name}}</option>
                 {{/each}}
             </select>
         </div>
     {{/if}}
 
-    {{#if typeRadio}}
+    {{#if typeRadioGrouped}}
         <div class="form-group radio {{key}}">
             <label for="{{key}}">{{name}}</label>
             {{#each options}}
                 <label class="label-radio">
-                    <input type="radio" name="{{../key}}" value="{{column}}" {{#if value}} checked {{/if}}> {{name}}
+                    <input type="radio" name="{{../key}}" value="{{key}}" {{#if value}} checked {{/if}}> {{name}}
                 </label>
             {{/each}}
         </div>
     {{/if}}
 
-    {{#if typeCheckbox}}
+    {{#if typeCheckboxGrouped}}
         <div class="form-group checkbox">
             <label>{{name}}</label>
             {{#each options}}
                 <label class="label-checkbox">
-                    <input type="checkbox" name="{{column}}" value="1" {{#if value}} checked {{/if}}> {{name}}
+                    <input type="checkbox" name="{{key}}" value="1" {{#if value}} checked {{/if}}> {{name}}
+                </label>
+            {{/each}}
+        </div>
+    {{/if}}
+
+    {{#if typeDropdownEnum }}
+        <div class="form-group dropdown {{key}}">
+            <label for="{{key}}">{{name}}</label>
+            <select name="{{key}}" class="form-control">
+                <option value="">
+                    –– {{#translate}}Select{{/translate}} ––
+                </option>
+                {{#each options}}
+                    <option value="{{key}}" {{#if value}} selected {{/if}}>{{name}}</option>
+                {{/each}}
+            </select>
+        </div>
+    {{/if}}
+
+    {{#if typeRadioEnum}}
+        <div class="form-group radio {{key}}">
+            <label for="{{key}}">{{name}}</label>
+            {{#each options}}
+                <label class="label-radio">
+                    <input type="radio" name="{{../key}}" value="{{key}}" {{#if value}} checked {{/if}}> {{name}}
                 </label>
             {{/each}}
         </div>

views/subscription/partials/subscription-manage-address-form.hbs

@@ -3,14 +3,14 @@
     <input type="hidden" name="_csrf" value="{{csrfToken}}">
     <input type="hidden" name="cid" value="{{cid}}">
 
-    <div class="form-group">
+    <div class="form-group email">
-        <label for="email">{{#translate}}Existing Email Address{{/translate}}</label>
+        <label for="EMAIL">{{#translate}}Existing Email Address{{/translate}}</label>
-        <input type="email" name="email" id="email" placeholder="" value="{{email}}" readonly>
+        <input type="email" name="EMAIL" id="email" placeholder="" value="{{email}}" readonly>
     </div>
 
-    <div class="form-group">
+    <div class="form-group email">
-        <label for="email-new">{{#translate}}New Email Address{{/translate}}</label>
+        <label for="EMAIL_NEW">{{#translate}}New Email Address{{/translate}}</label>
-        <input type="email" name="email-new" id="email-new" placeholder="{{#translate}}Your new email address{{/translate}}" value="{{email}}">
+        <input type="email" name="EMAIL_NEW" id="email-new" placeholder="{{#translate}}Your new email address{{/translate}}" value="{{email}}">
     </div>
 
     <p>