Lists list and CUD

Custom forms list
Updated DB schema (not yet implemented in the server, which means that most of the server is not broken).
- custom forms are independent of a list
- order and visibility of fields is now in custom_fields
- first_name and last_name has been turned to a regular custom field

models/forms.js

@@ -0,0 +1,261 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../shared/interoperable-errors');
+const shares = require('./shares');
+const bluebird = require('bluebird');
+const fs = bluebird.promisifyAll(require('fs'));
+const path = require('path');
+const mjml = require('mjml');
+const _ = require('../lib/translate')._;
+
+const formAllowedKeys = [
+    'name',
+    'description',
+    'layout',
+    'form_input_style'
+];
+
+const allowedFormKeys = [
+    'web_subscribe',
+    'web_confirm_subscription_notice',
+    'mail_confirm_subscription_html',
+    'mail_confirm_subscription_text',
+    'mail_already_subscribed_html',
+    'mail_already_subscribed_text',
+    'web_subscribed_notice',
+    'mail_subscription_confirmed_html',
+    'mail_subscription_confirmed_text',
+    'web_manage',
+    'web_manage_address',
+    'web_updated_notice',
+    'web_unsubscribe',
+    'web_confirm_unsubscription_notice',
+    'mail_confirm_unsubscription_html',
+    'mail_confirm_unsubscription_text',
+    'mail_confirm_address_change_html',
+    'mail_confirm_address_change_text',
+    'web_unsubscribed_notice',
+    'mail_unsubscription_confirmed_html',
+    'mail_unsubscription_confirmed_text',
+    'web_manual_unsubscribe_notice'
+];
+
+const hashKeys = [...formAllowedKeys, ...allowedFormKeys];
+
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'customForm', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('custom_forms')
+            .innerJoin('namespaces', 'namespaces.id', 'custom_forms.namespace'),
+        ['custom_forms.id', 'custom_forms.name', 'custom_forms.description', 'namespaces.name']
+    );
+}
+
+
+async function _getById(tx, id) {
+    const entity = await tx('custom_forms').where('id', id).first();
+
+    if (!entity) {
+        throw interoperableErrors.NotFoundError();
+    }
+
+    const forms = await tx('custom_forms_data').where('form', id).select(['data_key', 'data_value']);
+
+    for (const form of forms) {
+        entity[form.data_key] = form.data_value;
+    }
+
+    return entity;
+}
+
+
+async function getById(context, id) {
+    shares.enforceEntityPermission(context, 'customForm', id, 'view');
+
+    let entity;
+    await knex.transaction(async tx => {
+        entity = _getById(tx, id);
+    });
+
+    return entity;
+}
+
+
+async function serverValidate(context, data) {
+    const result = {};
+
+    const form = filterObject(data, allowedFormKeys);
+
+    const errs = checkForMjmlErrors(form);
+
+    for (const key in form) {
+        result[key] = {};
+        if (errs[key]) {
+            result.key.errors = errs[key];
+        }
+    }
+
+    return result;
+}
+
+
+async function create(context, entity) {
+    await shares.enforceEntityPermission(context, 'namespace', 'createCustomForm');
+
+    let id;
+    await knex.transaction(async tx => {
+        const ids = await tx('custom_forms').insert(filterObject(entity, formAllowedKeys));
+        id = ids[0];
+
+        const form = filterObject(entity, allowedFormKeys);
+        for (const formKey in form) {
+            await tx('custom_forms_data').insert({
+                form: id,
+                data_key: formKey,
+                data_value: form[formKey]
+            })
+        }
+    });
+
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await shares.enforceEntityPermission(context, 'customForm', entity.id, 'edit');
+
+    await knex.transaction(async tx => {
+        const existing = _getById(tx, context, id);
+
+        const existingHash = hash(existing);
+        if (existingHash != entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        const form = filterObject(entity, allowedFormKeys);
+        enforce(!Object.keys(checkForMjmlErrors(form)).length, 'Error(s) in form templates');
+
+        await tx('custom_forms').where('id', entity.id).update(filterObject(entity, formAllowedKeys));
+
+        for (const formKey in form) {
+            await tx('custom_forms_data').update({
+                form: entity.id,
+                data_key: formKey,
+                data_value: form[formKey]
+            });
+        }
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        const entity = await tx('custom_forms').where('id', id).first();
+
+        if (!entity) {
+            throw shares.throwPermissionDenied();
+        }
+
+        shares.enforceEntityPermission(context, 'list', entity.list, 'manageForms');
+
+        await tx('custom_forms_data').where('form', id).del();
+        await tx('custom_forms').where('id', id).del();
+    });
+}
+
+
+async function getDefaultFormValues() {
+    const basePath = path.join(__dirname, '..');
+
+    async function getContents(fileName) {
+        try {
+            const template = await fs.readFile(path.join(basePath, fileName), 'utf8');
+            return template.replace(/\{\{#translate\}\}(.*?)\{\{\/translate\}\}/g, (m, s) => _(s));
+        } catch (err) {
+            return false;
+        }
+    }
+
+    const form = {};
+
+    for (const key of allowedFormKeys) {
+        const base = 'views/subscription/' + key.replace(/_/g, '-');
+        if (key.startsWith('mail') || key.startsWith('web')) {
+            form[key] = await getContents(base + '.mjml.hbs') || await getContents(base + '.hbs') || '';
+        }
+    }
+
+    form.layout = await getContents('views/subscription/layout.mjml.hbs') || '';
+    form.formInputStyle = await getContents('public/subscription/form-input-style.css') || '@import url(/subscription/form-input-style.css);';
+
+    return form;
+}
+
+
+function checkForMjmlErrors(form) {
+    let testLayout = '<mjml><mj-body><mj-container>{{{body}}}</mj-container></mj-body></mjml>';
+
+    let hasMjmlError = (template, layout = testLayout) => {
+        let source = layout.replace(/\{\{\{body\}\}\}/g, template);
+        let compiled;
+
+        try {
+            compiled = mjml.mjml2html(source);
+        } catch (err) {
+            return err;
+        }
+
+        return compiled.errors;
+    };
+
+
+    const errors = {};
+    for (const key in form) {
+        if (key.startsWith('mail_') || key.startsWith('web_')) {
+            const template = form[key];
+            const errs = hasMjmlError(template);
+
+            if (key === 'mail_confirm_html' && !template.includes('{{confirmUrl}}')) {
+                errs.push('Missing {{confirmUrl}}');
+            }
+
+            if (errs.length) {
+                errors[key] = errs;
+            }
+
+        } else if (key === 'layout') {
+            const layout = values[index];
+            const err = hasMjmlError('', layout);
+
+            if (!layout.includes('{{{body}}}')) {
+                errs.push(`{{{body}}} not found`);
+            }
+
+            if (errs.length) {
+                errors[key] = errs;
+            }
+        }
+    }
+
+    return errors;
+}
+
+module.exports = {
+    listDTAjax,
+    hash,
+    getById,
+    create,
+    updateWithConsistencyCheck,
+    remove,
+    getDefaultFormValues,
+    serverValidate
+};

models/lists.js

@@ -1,14 +1,38 @@
 'use strict';
 
 const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
 const dtHelpers = require('../lib/dt-helpers');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
 const interoperableErrors = require('../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
 
-async function listDTAjax(params) {
-    return await dtHelpers.ajaxList(params, builder => builder.from('lists'), ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description']);
+const UnsubscriptionMode = require('../shared/lists').UnsubscriptionMode;
+
+const allowedKeys = new Set(['name', 'description', 'default_form', 'public_subscribe', 'unsubscription_mode', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
 }
 
-async function getById(id) {
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace'),
+        ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description', 'namespaces.name']
+    );
+}
+
+async function getById(context, id) {
+    shares.enforceEntityPermission(context, 'list', id, 'view');
+
     const entity = await knex('lists').where('id', id).first();
     if (!entity) {
         throw new interoperableErrors.NotFoundError();
@@ -17,8 +41,68 @@ async function getById(id) {
     return entity;
 }
 
+async function create(context, entity) {
+    await shares.enforceEntityPermission(context, 'namespace', entity.namespace, 'createList');
+
+    let id;
+    await knex.transaction(async tx => {
+        await namespaceHelpers.validateEntity(tx, entity);
+        enforce(entity.unsubscription_mode >= 0 && entity.unsubscription_mode < UnsubscriptionMode.MAX, 'Unknown unsubscription mode');
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.cid = shortid.generate();
+
+        const ids = await tx('lists').insert(filteredEntity);
+        id = ids[0];
+
+        await knex.schema.raw('CREATE TABLE `subscription__' + id + '` LIKE subscription');
+
+        await shares.rebuildPermissions(tx, { entityTypeId: 'list', entityId: id });
+    });
+
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await shares.enforceEntityPermission(context, 'list', entity.id, 'edit');
+
+    await knex.transaction(async tx => {
+        const existing = await tx('lists').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash != entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'list', 'createList', 'delete');
+        enforce(entity.unsubscription_mode >= 0 && entity.unsubscription_mode < UnsubscriptionMode.MAX, 'Unknown unsubscription mode');
+
+        await tx('lists').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissions(tx, { entityTypeId: 'list', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await shares.enforceEntityPermission(context, 'list', id, 'delete');
+
+    await knex.transaction(async tx => {
+        await tx('lists').where('id', id).del();
+        await knex.schema.dropTableIfExists('subscription__' + id);
+    });
+}
+
 
 module.exports = {
+    UnsubscriptionMode,
+    hash,
     listDTAjax,
-    getById
+    getById,
+    create,
+    updateWithConsistencyCheck,
+    remove
 };

models/namespaces.js

@@ -151,6 +151,9 @@ async function updateWithConsistencyCheck(context, entity) {
             throw new interoperableErrors.ChangedError();
         }
 
+        // namespaceHelpers.validateEntity is not needed here because it is part of the tree traversal check below
+        await namespaceHelpers.validateMove(context, entity, existing, 'namespace', 'createNamespace', 'delete');
+
         let iter = entity;
         while (iter.namespace != null) {
             iter = await tx('namespaces').where('id', iter.namespace).first();

models/report-templates.js

@@ -66,11 +66,7 @@ async function updateWithConsistencyCheck(context, entity) {
         }
 
         await namespaceHelpers.validateEntity(tx, entity);
-
-        if (existing.namespace !== entity.namespace) {
-            await shares.enforceEntityPermission(context, 'namespace', entity.namespace, 'createReport');
-            await shares.enforceEntityPermission(context, 'reportTemplate', entity.id, 'delete');
-        }
+        await namespaceHelpers.validateMove(context, entity, existing, 'reportTemplate', 'createReportTemplate', 'delete');
 
         await tx('report_templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
 

models/reports.js

@@ -19,9 +19,7 @@ function hash(entity) {
 }
 
 async function getByIdWithTemplate(context, id) {
-    if (context) {
-        await shares.enforceEntityPermission(context, 'report', id, 'view');
-    }
+    await shares.enforceEntityPermission(context, 'report', id, 'view');
 
     const entity = await knex('reports')
         .where('reports.id', id)
@@ -104,11 +102,7 @@ async function updateWithConsistencyCheck(context, entity) {
         }
 
         await namespaceHelpers.validateEntity(tx, entity);
-
-        if (existing.namespace !== entity.namespace) {
-            await shares.enforceEntityPermission(context, 'namespace', entity.namespace, 'createReport');
-            await shares.enforceEntityPermission(context, 'report', entity.id, 'delete');
-        }
+        await namespaceHelpers.validateMove(context, entity, existing, 'report', 'createReport', 'delete');
 
         entity.params = JSON.stringify(entity.params);
 

models/shares.js

@@ -22,7 +22,7 @@ async function listByEntityDTAjax(context, entityTypeId, entityId, params) {
             .innerJoin('generated_role_names', 'generated_role_names.role', 'users.role')
             .where('generated_role_names.entity_type', entityTypeId)
             .where(`${entityType.sharesTable}.entity`, entityId),
-        [ 'users.username', 'users.name', 'generated_role_names.name', 'users.id' ]
+        [ 'users.username', 'users.name', 'generated_role_names.name', 'users.id', entityType.sharesTable + '.auto' ]
     );
 }
 
@@ -46,7 +46,7 @@ async function listByUserDTAjax(context, entityTypeId, userId, params) {
             .innerJoin('generated_role_names', 'generated_role_names.role', entityType.sharesTable + '.role')
             .where('generated_role_names.entity_type', entityTypeId)
             .where(entityType.sharesTable + '.user', userId),
-        [ entityType.entitiesTable + '.name', 'generated_role_names.name', entityType.entitiesTable + '.id' ]
+        [ entityType.entitiesTable + '.name', 'generated_role_names.name', entityType.entitiesTable + '.id', entityType.sharesTable + '.auto' ]
     );
 }
 
@@ -167,7 +167,7 @@ async function _rebuildPermissions(tx, restriction) {
             const desiredRole = roleConf.ownNamespaceRole;
             if (desiredRole && user.role !== desiredRole) {
                 await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: user.namespace }).del();
-                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: user.namespace, role: desiredRole });
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: user.namespace, role: desiredRole, auto: true });
             }
         }
     }
@@ -191,7 +191,7 @@ async function _rebuildPermissions(tx, restriction) {
             const desiredRole = roleConf.rootNamespaceRole;
             if (desiredRole && user.role !== desiredRole) {
                 await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: 1 /* Global namespace id */ }).del();
-                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: 1 /* Global namespace id */, role: desiredRole });
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: 1 /* Global namespace id */, role: desiredRole, auto: 1 });
             }
         }
     }
@@ -268,7 +268,7 @@ async function _rebuildPermissions(tx, restriction) {
                         }
                     }
                 } else {
-                    const userPerms = {}
+                    const userPerms = {};
                     ns.transitiveUserPermissions.set(user, userPerms);
 
                     for (const entityTypeId in restrictedEntityTypes) {
@@ -407,6 +407,10 @@ async function removeDefaultShares(tx, user) {
 }
 
 function enforceGlobalPermission(context, requiredOperations) {
+    if (context.user.admin) { // This handles the getAdminContext() case
+        return;
+    }
+
     if (typeof requiredOperations === 'string') {
         requiredOperations = [ requiredOperations ];
     }
@@ -424,6 +428,10 @@ function enforceGlobalPermission(context, requiredOperations) {
 }
 
 async function _checkPermission(context, entityTypeId, entityId, requiredOperations) {
+    if (context.user.admin) { // This handles the getAdminContext() case
+        return true;
+    }
+
     const entityType = permissions.getEntityType(entityTypeId);
 
     if (typeof requiredOperations === 'string') {

models/users.js

@@ -31,7 +31,7 @@ const ownAccountAllowedKeys = new Set(['name', 'email', 'password']);
 const allowedKeysExternal = new Set(['username', 'namespace', 'role']);
 const hashKeys = new Set(['username', 'name', 'email', 'namespace', 'role']);
 const shares = require('./shares');
-
+const contextHelpers = require('../lib/context-helpers');
 
 function hash(entity) {
     return hasher.hash(filterObject(entity, hashKeys));
@@ -54,9 +54,7 @@ async function _getBy(context, key, value, extraColumns) {
         }
     }
 
-    if (context) {
-        await shares.enforceEntityPermission(context, 'namespace', user.namespace, 'manageUsers');
-    }
+    await shares.enforceEntityPermission(context, 'namespace', user.namespace, 'manageUsers');
 
     return user;
 }
@@ -260,16 +258,16 @@ async function remove(context, userId) {
 }
 
 async function getByAccessToken(accessToken) {
-    return await _getBy(null, 'access_token', accessToken);
+    return await _getBy(contextHelpers.getAdminContext(), 'access_token', accessToken);
 }
 
 async function getByUsername(username) {
-    return await _getBy(null, 'username', username);
+    return await _getBy(contextHelpers.getAdminContext(), 'username', username);
 }
 
 async function getByUsernameIfPasswordMatch(username, password) {
     try {
-        const user = await _getBy(null, 'username', username, ['password']);
+        const user = await _getBy(contextHelpers.getAdminContext(), 'username', username, ['password']);
 
         if (!await bcryptCompare(password, user.password)) {
             throw new interoperableErrors.IncorrectPasswordError();
@@ -287,7 +285,7 @@ async function getByUsernameIfPasswordMatch(username, password) {
 }
 
 async function getAccessToken(userId) {
-    const user = await _getBy(null, 'id', userId, ['access_token']);
+    const user = await _getBy(contextHelpers.getAdminContext(), 'id', userId, ['access_token']);
     return user.access_token;
 }