Lists list and CUD

Custom forms list
Updated DB schema (not yet implemented in the server, which means that most of the server is not broken).
- custom forms are independent of a list
- order and visibility of fields is now in custom_fields
- first_name and last_name has been turned to a regular custom field

access-control.md

@@ -60,9 +60,18 @@ A namespace role further defines allowed operations for entity types within and
 
 The following defines the role master for scope "global". This effectively means that in 
 "Create/Edit User" form, the user can be given role "Master". 
-The role gives the permission to rebuild the permission cache. Further, it specifies that the
+The role gives the permission to rebuild the permission cache. 
+
+Further, it specifies that the
 holder of the role will automatically be given access (share) to the root namespace in the
-namespace role "master". This role is also an admin role, which means that user id 1 will always be reset to this role.
+namespace role "master" (specified by ```rootNamespaceRole="master"```). This access to the root namespace is given irrespective of the namespace
+in which the user is created. This highlight the dual purpose of namespaces: a) they group
+entities w.r.t. access management, b) they allow categorizing entities and users in a hierarchy
+to potentially reflect the organisational or process hierarchy. The latter is especially useful for
+more enterprise applications where a single installation of Mailtrain serves a number of rather
+independent groups.
+
+The global role defined below is also an admin role (denoted by the ```admin=true```), which means that user id 1 will always be reset to this role.
 This serves as a kind of bootstrap that makes sure that there is always a user that can be
 used to give access to other users.
 ```
@@ -74,11 +83,23 @@ permissions=["rebuildPermissions"]
 rootNamespaceRole="master"
 ```
 
-This defines the role "master" for "report" entities. It lists the operations that a user
+Another example for a global role is the following. This one is intended for regular users.
+As such, it does not automatically give access to everything. Rather, it gives limited access
+to entities under the namespace in which the user has been created. This is specified by the
+```ownNamespaceRole="editor"```
+```
+[roles.global.editor]
+name="Editor"
+description="Anything under own namespace except operations related to sending and doing reports"
+permissions=[]
+ownNamespaceRole="editor"
+```
+
+The roles for entities are defined in a similar fashion. The example below shows the definition
+of the role "master" for "report" entities. It lists the operations that a user
 that has "master" access to a particular report can do with the report. Note that to get the
 "master" access to a particular report through this role, the report would either have to be shared with the user
 with role "master".  
-  
 ```
 [roles.report.master]
 name="Master"
@@ -86,6 +107,14 @@ description="All permissions"
 permissions=["view", "edit", "delete", "share", "execute", "viewContent", "viewOutput"]
 ```
 
+The same for the restricted role "editor" can look as follows.
+```
+[roles.report.editor]
+name="Editor"
+description="Anything under own namespace except operations related to sending and doing reports"
+permissions=["view", "viewContent", "viewOutput"]
+```
+
 The following defines the role "master" for "namespace" entities. Similarly to the example above,
 it lists operations that relate to a namespace. In particular all "create" operations pertain
 to a namespace rathen than to an entity, which at the time of creation does not exist yet.
@@ -102,3 +131,11 @@ reportTemplate=["view", "edit", "delete", "share", "execute"]
 report=["view", "edit", "delete", "share", "execute", "viewContent", "viewOutput"]
 namespace=["view", "edit", "delete", "share", "createNamespace", "createReportTemplate", "createReport", "manageUsers"]
 ```
+
+And the same for the more restricted role "editor".
+```
+[roles.namespace.editor.children]
+reportTemplate=[]
+report=["view", "viewContent", "viewOutput"]
+namespace=["view", "edit", "delete"]
+```

app.js

@@ -22,14 +22,14 @@ const tools = require('./lib/tools');
 const contextHelpers = require('./lib/context-helpers');
 
 const routes = require('./routes/index');
-const lists = require('./routes/lists');
+const lists = require('./routes/lists-legacy');
 const settings = require('./routes/settings');
 const settingsModel = require('./lib/models/settings');
 const templates = require('./routes/templates');
 const campaigns = require('./routes/campaigns');
 const links = require('./routes/links');
 const fields = require('./routes/fields');
-const forms = require('./routes/forms');
+const forms = require('./routes/forms-legacy');
 const segments = require('./routes/segments');
 const triggers = require('./routes/triggers');
 const webhooks = require('./routes/webhooks');
@@ -51,12 +51,14 @@ const reportTemplatesRest = require('./routes/rest/report-templates');
 const reportsRest = require('./routes/rest/reports');
 const campaignsRest = require('./routes/rest/campaigns');
 const listsRest = require('./routes/rest/lists');
+const formsRest = require('./routes/rest/forms');
 const sharesRest = require('./routes/rest/shares');
 
 const namespacesLegacyIntegration = require('./routes/namespaces-legacy-integration');
 const usersLegacyIntegration = require('./routes/users-legacy-integration');
 const accountLegacyIntegration = require('./routes/account-legacy-integration');
 const reportsLegacyIntegration = require('./routes/reports-legacy-integration');
+const listsLegacyIntegration = require('./routes/lists-legacy-integration');
 
 const interoperableErrors = require('./shared/interoperable-errors');
 
@@ -253,6 +255,7 @@ if (config.reports && config.reports.enabled === true) {
 app.use('/users', usersLegacyIntegration);
 app.use('/namespaces', namespacesLegacyIntegration);
 app.use('/account', accountLegacyIntegration);
+app.use('/lists', listsLegacyIntegration);
 
 if (config.reports && config.reports.enabled === true) {
     app.use('/reports', reports);
@@ -270,6 +273,7 @@ app.use('/rest', usersRest);
 app.use('/rest', accountRest);
 app.use('/rest', campaignsRest);
 app.use('/rest', listsRest);
+app.use('/rest', formsRest);
 app.use('/rest', sharesRest);
 
 if (config.reports && config.reports.enabled === true) {
@@ -329,6 +333,7 @@ if (app.get('env') === 'development') {
             return next();
         }
 
+        console.log(err);
         if (req.needsJSONResponse) {
             const resp = {
                 message: err.message,

client/src/account/Login.js

@@ -105,7 +105,7 @@ export default class Login extends Component {
                 <Form stateOwner={this} onSubmitAsync={::this.submitHandler}>
                     <InputField id="username" label={t('Username')}/>
                     <InputField id="password" label={t('Password')} type="password" />
-                    <CheckBox id="remember" label={t('Remember me')}/>
+                    <CheckBox id="remember" text={t('Remember me')}/>
 
                     <ButtonRow>
                         <Button type="submit" className="btn-primary" icon="ok" label={t('Sign in')}/>

client/src/lib/form.js

@@ -109,7 +109,7 @@ function wrapInput(id, htmlId, owner, label, help, input) {
     const helpBlock = help ? <div className="help-block col-sm-offset-2 col-sm-10" id={htmlId + '_help'}>{help}</div> : '';
 
     return (
-        <div className={owner.addFormValidationClass('form-group', id)} >
+        <div className={id ? owner.addFormValidationClass('form-group', id) : 'form-group'} >
             <div className="col-sm-2">
                 <label htmlFor={htmlId} className="control-label">{label}</label>
             </div>
@@ -117,25 +117,47 @@ function wrapInput(id, htmlId, owner, label, help, input) {
                 {input}
             </div>
             {helpBlock}
-            <div className="help-block col-sm-offset-2 col-sm-10" id={htmlId + '_help_validation'}>{owner.getFormValidationMessage(id)}</div>
+            {id && <div className="help-block col-sm-offset-2 col-sm-10" id={htmlId + '_help_validation'}>{owner.getFormValidationMessage(id)}</div>}
         </div>
     );
 }
 
-function wrapInputInline(id, htmlId, owner, containerClass, label, help, input) {
+function wrapInputInline(id, htmlId, owner, containerClass, label, text, help, input) {
     const helpBlock = help ? <div className="help-block col-sm-offset-2 col-sm-10" id={htmlId + '_help'}>{help}</div> : '';
 
     return (
-        <div className={owner.addFormValidationClass('form-group', id)} >
+        <div className={id ? owner.addFormValidationClass('form-group', id) : 'form-group'} >
-            <div className={"col-sm-10 col-sm-offset-2 " + containerClass }>
+            <div className="col-sm-2">
-                <label>{input} {label}</label>
+                <label className="control-label">{label}</label>
+            </div>
+            <div className={"col-sm-10 " + containerClass }>
+                <label>{input} {text}</label>
             </div>
             {helpBlock}
-            <div className="help-block col-sm-offset-2 col-sm-10" id={htmlId + '_help_validation'}>{owner.getFormValidationMessage(id)}</div>
+            {id && <div className="help-block col-sm-offset-2 col-sm-10" id={htmlId + '_help_validation'}>{owner.getFormValidationMessage(id)}</div>}
         </div>
     );
 }
 
+class StaticField extends Component {
+    static propTypes = {
+        id: PropTypes.string.isRequired,
+        label: PropTypes.string.isRequired,
+        help: PropTypes.oneOfType([PropTypes.string, PropTypes.object])
+    }
+
+    render() {
+        const props = this.props;
+        const owner = this.context.formStateOwner;
+        const id = this.props.id;
+        const htmlId = 'form_' + id;
+
+        return wrapInput(null, htmlId, owner, props.label, props.help,
+            <div id={htmlId} className="form-control" aria-describedby={htmlId + '_help'}>{props.children}</div>
+        );
+    }
+}
+
 class InputField extends Component {
     static propTypes = {
         id: PropTypes.string.isRequired,
@@ -173,7 +195,8 @@ class InputField extends Component {
 class CheckBox extends Component {
     static propTypes = {
         id: PropTypes.string.isRequired,
-        label: PropTypes.string.isRequired,
+        text: PropTypes.string.isRequired,
+        label: PropTypes.string,
         help: PropTypes.oneOfType([PropTypes.string, PropTypes.object])
     }
 
@@ -187,8 +210,8 @@ class CheckBox extends Component {
         const id = this.props.id;
         const htmlId = 'form_' + id;
 
-        return wrapInputInline(id, htmlId, owner, 'checkbox', props.label, props.help,
+        return wrapInputInline(id, htmlId, owner, 'checkbox', props.label, props.text, props.help,
-            <input type="checkbox" checked={owner.getFormValue(id)} id={htmlId} aria-describedby={htmlId + '_help'} onClick={evt => owner.updateFormValue(id, !owner.getFormValue(id))}/>
+            <input type="checkbox" checked={owner.getFormValue(id)} id={htmlId} aria-describedby={htmlId + '_help'} onChange={evt => owner.updateFormValue(id, !owner.getFormValue(id))}/>
         );
     }
 }
@@ -864,6 +887,7 @@ export {
     withForm,
     Form,
     Fieldset,
+    StaticField,
     InputField,
     CheckBox,
     TextArea,

client/src/lib/table.js

@@ -239,6 +239,8 @@ class Table extends Component {
                 type: 'html',
                 createdCell: createdCellFn
             });
+
+            // FIXME, sift all columns through renderToStaticMarkup in order to sanitize the HTML
         }
 
         const dtOptions = {

client/src/lib/tree.js

@@ -109,6 +109,8 @@ class TreeTable extends Component {
 
             let tdIdx = 1;
 
+            // FIXME, sift title through renderToStaticMarkup in order to sanitize the HTML
+
             if (this.props.withDescription) {
                 const descHtml = ReactDOMServer.renderToStaticMarkup(<div>{node.data.description}</div>);
                 tdList.eq(tdIdx).html(descHtml);

client/src/lists/CUD.js

@@ -0,0 +1,228 @@
+'use strict';
+
+import React, { Component } from 'react';
+import PropTypes from 'prop-types';
+import { translate, Trans } from 'react-i18next';
+import { requiresAuthenticatedUser, withPageHelpers, Title } from '../lib/page';
+import {
+    withForm, Form, FormSendMethod, InputField, TextArea, TableSelect, TableSelectMode, ButtonRow, Button,
+    Fieldset, Dropdown, AlignedRow, StaticField, CheckBox
+} from '../lib/form';
+import axios from '../lib/axios';
+import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
+import { ModalDialog } from '../lib/bootstrap-components';
+import { validateNamespace, NamespaceSelect } from '../lib/namespace';
+import { UnsubscriptionMode } from '../../../shared/lists';
+
+@translate()
+@withForm
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class CUD extends Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            customFormOptions: []
+        };
+
+        if (props.edit) {
+            this.state.entityId = parseInt(props.match.params.id);
+        }
+
+        this.initForm();
+    }
+
+    static propTypes = {
+        edit: PropTypes.bool
+    }
+
+    isDelete() {
+        return this.props.match.params.action === 'delete';
+    }
+
+    @withAsyncErrorHandler
+    async loadFormValues() {
+        await this.getFormValuesFromURL(`/rest/lists/${this.state.entityId}`, data => {
+            data.form = data.default_form ? 'custom' : 'default';
+        });
+    }
+
+    componentDidMount() {
+        if (this.props.edit) {
+            this.loadFormValues();
+        } else {
+            this.populateFormValues({
+                name: '',
+                description: '',
+                form: 'default',
+                default_form: null,
+                public_subscribe: true,
+                unsubscription_mode: UnsubscriptionMode.ONE_STEP,
+                namespace: null
+            });
+        }
+    }
+
+    localValidateFormValues(state) {
+        const t = this.props.t;
+        const edit = this.props.edit;
+
+        if (!state.getIn(['name', 'value'])) {
+            state.setIn(['name', 'error'], t('Name must not be empty'));
+        } else {
+            state.setIn(['name', 'error'], null);
+        }
+
+        if (state.getIn(['form', 'value']) === 'custom' && !state.getIn(['default_form', 'value'])) {
+            state.setIn(['default_form', 'error'], t('Custom form must be selected'));
+        } else {
+            state.setIn(['default_form', 'error'], null);
+        }
+
+        validateNamespace(t, state);
+    }
+
+    async submitHandler() {
+        const t = this.props.t;
+        const edit = this.props.edit;
+
+        let sendMethod, url;
+        if (edit) {
+            sendMethod = FormSendMethod.PUT;
+            url = `/rest/lists/${this.state.entityId}`
+        } else {
+            sendMethod = FormSendMethod.POST;
+            url = '/rest/lists'
+        }
+
+        this.disableForm();
+        this.setFormStatusMessage('info', t('Saving list ...'));
+
+        const submitSuccessful = await this.validateAndSendFormValuesToURL(sendMethod, url, data => {
+            if (data.form === 'default') {
+                data.default_form = null;
+            }
+        });
+
+        if (submitSuccessful) {
+            this.navigateToWithFlashMessage('/lists', 'success', t('List saved'));
+        } else {
+            this.enableForm();
+            this.setFormStatusMessage('warning', t('There are errors in the form. Please fix them and submit again.'));
+        }
+    }
+
+    async showDeleteModal() {
+        this.navigateTo(`/lists/edit/${this.state.entityId}/delete`);
+    }
+
+    async hideDeleteModal() {
+        this.navigateTo(`/lists/edit/${this.state.entityId}`);
+    }
+
+    async performDelete() {
+        const t = this.props.t;
+
+        await this.hideDeleteModal();
+
+        this.disableForm();
+        this.setFormStatusMessage('info', t('Deleting list...'));
+
+        await axios.delete(`/rest/lists/${this.state.entityId}`);
+
+        this.navigateToWithFlashMessage('/lists', 'success', t('List deleted'));
+    }
+
+    render() {
+        const t = this.props.t;
+        const edit = this.props.edit;
+
+        const unsubcriptionModeOptions = [
+            {
+                key: UnsubscriptionMode.ONE_STEP,
+                label: t('One-step (i.e. no email with confirmation link)')
+            },
+            {
+                key: UnsubscriptionMode.ONE_STEP_WITH_FORM,
+                label: t('One-step with unsubscription form (i.e. no email with confirmation link)')
+            },
+            {
+                key: UnsubscriptionMode.TWO_STEP,
+                label: t('Two-step (i.e. an email with confirmation link will be sent)')
+            },
+            {
+                key: UnsubscriptionMode.TWO_STEP_WITH_FORM,
+                label: t('Two-step with unsubscription form (i.e. an email with confirmation link will be sent)')
+            },
+            {
+                key: UnsubscriptionMode.MANUAL,
+                label: t('Manual (i.e. unsubscription has to be performed by the list administrator)')
+            }
+        ];
+
+        const formsOptions = [
+            {
+                key: 'default',
+                label: t('Default Mailtrain Forms')
+            },
+            {
+                key: 'custom',
+                label: t('Custom Forms (select form below)')
+            }
+        ]
+
+        const customFormsColumns = [
+            {data: 0, title: "#"},
+            {data: 1, title: t('Name')},
+            {data: 2, title: t('Description')},
+            {data: 3, title: t('Namespace')}
+        ];
+
+        return (
+            <div>
+                {edit &&
+                    <ModalDialog hidden={!this.isDelete()} title={t('Confirm deletion')} onCloseAsync={::this.hideDeleteModal} buttons={[
+                        { label: t('No'), className: 'btn-primary', onClickAsync: ::this.hideDeleteModal },
+                        { label: t('Yes'), className: 'btn-danger', onClickAsync: ::this.performDelete }
+                    ]}>
+                        {t('Are you sure you want to delete "{{name}}"?', {name: this.getFormValue('name')})}
+                    </ModalDialog>
+                }
+
+                <Title>{edit ? t('Edit List') : t('Create List')}</Title>
+
+                <Form stateOwner={this} onSubmitAsync={::this.submitHandler}>
+                    <InputField id="name" label={t('Name')}/>
+
+                    {edit &&
+                        <StaticField id="cid" label="List ID" help={t('This is the list ID displayed to the subscribers')}>
+                            {this.getFormValue('cid')}
+                        </StaticField>
+                    }
+
+                    <TextArea id="description" label={t('Description')} help={t('HTML is allowed')}/>
+
+                    <NamespaceSelect/>
+
+                    <Dropdown id="form" label={t('Forms')} options={formsOptions} help={t('Web and email forms and templates used in subscription management process.')}/>
+
+                    {this.getFormValue('form') === 'custom' &&
+                        <TableSelect id="default_form" label={t('Custom Forms')} withHeader dropdown dataUrl='/rest/forms-table' columns={customFormsColumns} selectionLabelIndex={1} help={<Trans>The custom form used for this list. You can create a form <a href={`/lists/forms/create/${this.state.entityId}`}>here</a>.</Trans>}/>
+                    }
+
+                    <CheckBox id="public_subscribe" label={t('Subscription')} text={t('Allow public users to subscribe themselves')}/>
+
+                    <Dropdown id="unsubscription_mode" label={t('Unsubscription')} options={unsubcriptionModeOptions} help={t('Select how an unsuscription request by subscriber is handled.')}/>
+
+
+                    <ButtonRow>
+                        <Button type="submit" className="btn-primary" icon="ok" label={t('Save')}/>
+                        {edit && <Button className="btn-danger" icon="remove" label={t('Delete List')} onClickAsync={::this.showDeleteModal}/>}
+                    </ButtonRow>
+                </Form>
+            </div>
+        );
+    }
+}

client/src/lists/List.js

@@ -0,0 +1,89 @@
+'use strict';
+
+import React, { Component } from 'react';
+import { translate } from 'react-i18next';
+import {requiresAuthenticatedUser, withPageHelpers, Title, Toolbar, NavButton} from '../lib/page';
+import { withErrorHandling, withAsyncErrorHandler } from '../lib/error-handling';
+import { Table } from '../lib/table';
+import axios from '../lib/axios';
+
+@translate()
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class List extends Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {};
+    }
+
+    @withAsyncErrorHandler
+    async fetchPermissions() {
+        const request = {
+            createList: {
+                entityTypeId: 'namespace',
+                requiredOperations: ['createList']
+            }
+        };
+
+        const result = await axios.post('/rest/permissions-check', request);
+
+        this.setState({
+            createPermitted: result.data.createList
+        });
+    }
+
+    componentDidMount() {
+        this.fetchPermissions();
+    }
+
+    render() {
+        const t = this.props.t;
+
+        const actions = data => {
+            const actions = [];
+            const perms = data[6];
+
+            if (perms.includes('edit')) {
+                actions.push({
+                    label: <span className="glyphicon glyphicon-edit" aria-hidden="true" title="Edit"></span>,
+                    link: '/lists/edit/' + data[0]
+                });
+            }
+
+            if (perms.includes('share')) {
+                actions.push({
+                    label: <span className="glyphicon glyphicon-share-alt" aria-hidden="true" title="Share"></span>,
+                    link: '/lists/share/' + data[0]
+                });
+            }
+
+            return actions;
+        };
+
+        const columns = [
+            { data: 0, title: "#" },
+            { data: 1, title: t('Name') },
+            { data: 2, title: t('ID'), render: data => `<code>${data}</code>` },
+            { data: 3, title: t('Subscribers') },
+            { data: 4, title: t('Description') },
+            { data: 5, title: t('Namespace') }
+        ];
+
+        return (
+            <div>
+                {this.state.createPermitted &&
+                    <Toolbar>
+                        <NavButton linkTo="/lists/create" className="btn-primary" icon="plus" label={t('Create List')}/>
+                        <NavButton linkTo="/lists/forms" className="btn-primary" label={t('Custom Forms')}/>
+                    </Toolbar>
+                }
+
+                <Title>{t('Lists')}</Title>
+
+                <Table withHeader dataUrl="/rest/lists-table" columns={columns} actions={actions} />
+            </div>
+        );
+    }
+}

client/src/lists/forms/List.js

@@ -0,0 +1,85 @@
+'use strict';
+
+import React, { Component } from 'react';
+import { translate } from 'react-i18next';
+import {requiresAuthenticatedUser, withPageHelpers, Title, Toolbar, NavButton} from '../../lib/page';
+import { withErrorHandling, withAsyncErrorHandler } from '../../lib/error-handling';
+import { Table } from '../../lib/table';
+import axios from '../../lib/axios';
+
+@translate()
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class List extends Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {};
+    }
+
+    @withAsyncErrorHandler
+    async fetchPermissions() {
+        const request = {
+            createCustomForm: {
+                entityTypeId: 'namespace',
+                requiredOperations: ['createCustomForm']
+            }
+        };
+
+        const result = await axios.post('/rest/permissions-check', request);
+
+        this.setState({
+            createPermitted: result.data.createCustomForm
+        });
+    }
+
+    componentDidMount() {
+        this.fetchPermissions();
+    }
+
+    render() {
+        const t = this.props.t;
+
+        const actions = data => {
+            const actions = [];
+            const perms = data[4];
+
+            if (perms.includes('edit')) {
+                actions.push({
+                    label: <span className="glyphicon glyphicon-edit" aria-hidden="true" title="Edit"></span>,
+                    link: '/lists/forms/edit/' + data[0]
+                });
+            }
+            if (perms.includes('share')) {
+                actions.push({
+                    label: <span className="glyphicon glyphicon-share-alt" aria-hidden="true" title="Share"></span>,
+                    link: '/lists/forms/share/' + data[0]
+                });
+            }
+
+            return actions;
+        };
+
+        const columns = [
+            { data: 0, title: "#" },
+            { data: 1, title: t('Name') },
+            { data: 2, title: t('Description') },
+            { data: 3, title: t('Namespace') }
+        ];
+
+        return (
+            <div>
+                {this.state.createPermitted &&
+                    <Toolbar>
+                        <NavButton linkTo="/lists/forms/create" className="btn-primary" icon="plus" label={t('Create Custom Form')}/>
+                    </Toolbar>
+                }
+
+                <Title>{t('Forms')}</Title>
+
+                <Table withHeader dataUrl="/rest/forms-table" columns={columns} actions={actions} />
+            </div>
+        );
+    }
+}

client/src/lists/root.js

@@ -0,0 +1,61 @@
+'use strict';
+
+import React from 'react';
+import ReactDOM from 'react-dom';
+import { I18nextProvider } from 'react-i18next';
+import i18n from '../lib/i18n';
+
+import { Section } from '../lib/page';
+import ListsList from './List';
+import ListsCUD from './CUD';
+import FormsList from './forms/List';
+import Share from '../shares/Share';
+
+
+const getStructure = t => {
+    const subPaths = {};
+
+    return {
+        '': {
+            title: t('Home'),
+            externalLink: '/',
+            children: {
+                'lists': {
+                    title: t('Lists'),
+                    link: '/lists',
+                    component: ListsList,
+                    children: {
+                        edit: {
+                            title: t('Edit List'),
+                            params: [':id', ':action?'],
+                            render: props => (<ListsCUD edit {...props} />)
+                        },
+                        create: {
+                            title: t('Create List'),
+                            render: props => (<ListsCUD {...props} />)
+                        },
+                        share: {
+                            title: t('Share List'),
+                            params: [':id'],
+                            render: props => (<Share title={entity => t('Share List "{{name}}"', {name: entity.name})} getUrl={id => `/rest/lists/${id}`} entityTypeId="list" {...props} />)
+                        },
+                        forms: {
+                            title: t('Forms'),
+                            link: '/lists/forms',
+                            component: FormsList,
+                        }
+                    }
+                }
+            }
+        }
+    }
+};
+
+export default function() {
+    ReactDOM.render(
+        <I18nextProvider i18n={ i18n }><Section root='/lists' structure={getStructure}/></I18nextProvider>,
+        document.getElementById('root')
+    );
+};
+
+

client/src/reports/CUD.js

@@ -147,7 +147,7 @@ export default class CUD extends Component {
         }
 
         this.disableForm();
-        this.setFormStatusMessage('info', t('Saving report template ...'));
+        this.setFormStatusMessage('info', t('Saving report ...'));
 
         const submitSuccessful = await this.validateAndSendFormValuesToURL(sendMethod, url, data => {
             const params = {};
@@ -245,8 +245,6 @@ export default class CUD extends Component {
             }
         }
 
-        // FIXME - filter namespaces by permission
-
         return (
             <div>
                 {edit &&

client/src/reports/Output.js

@@ -22,8 +22,9 @@ export default class Output extends Component {
     @withAsyncErrorHandler
     async loadOutput() {
         const id = parseInt(this.props.match.params.id);
-        const outputResp = await axios.get(`/rest/report-output/${id}`);
+        const outputRespPromise = axios.get(`/rest/report-output/${id}`);
-        const reportResp = await axios.get(`/rest/reports/${id}`);
+        const reportRespPromise = axios.get(`/rest/reports/${id}`);
+        const [outputResp, reportResp] = await Promise.all([outputRespPromise, reportRespPromise]);
 
         this.setState({
             output: outputResp.data,

client/src/reports/View.js

@@ -23,8 +23,9 @@ export default class View extends Component {
     @withAsyncErrorHandler
     async loadContent() {
         const id = parseInt(this.props.match.params.id);
-        const contentResp = await axios.get(`/rest/report-content/${id}`);
+        const contentRespPromise = axios.get(`/rest/report-content/${id}`);
-        const reportResp = await axios.get(`/rest/reports/${id}`);
+        const reportRespPromise = axios.get(`/rest/reports/${id}`);
+        const [contentResp, reportResp] = await Promise.all([contentRespPromise, reportRespPromise]);
 
         this.setState({
             content: contentResp.data,

client/src/reports/templates/CUD.js

@@ -301,8 +301,6 @@ export default class CUD extends Component {
         const t = this.props.t;
         const edit = this.props.edit;
 
-        // FIXME - filter namespaces by permission
-
         return (
             <div>
                 {edit &&

client/src/shares/Share.js

@@ -111,12 +111,19 @@ export default class Share extends Component {
     render() {
         const t = this.props.t;
 
-        const actions = data => [
+        const actions = data => {
-            {
+            const actions = [];
+            const autoGenerated = data[4];
+
+            if (!autoGenerated) {
+                actions.push({
                     label: 'Delete',
                     action: () => this.deleteShare(data[3])
+                });
             }
-        ];
+
+            return actions;
+        };
 
         const sharesColumns = [];
         sharesColumns.push({ data: 0, title: t('Username') });

client/src/shares/UserShares.js

@@ -53,9 +53,10 @@ export default class UserShares extends Component {
         const renderSharesTable = (entityTypeId, title) => {
             const actions = data => {
                 const actions = [];
-                const perms = data[3];
+                const autoGenerated = data[3];
+                const perms = data[4];
 
-                if (perms.includes('share')) {
+                if (!autoGenerated && perms.includes('share')) {
                     actions.push({
                         label: 'Delete',
                         action: () => this.deleteShare(entityTypeId, data[2])
@@ -85,8 +86,10 @@ export default class UserShares extends Component {
                 <Title>{t('Shares for user "{{username}}"', {username: this.state.username})}</Title>
 
                 {renderSharesTable('namespace', t('Namespaces'))}
-                {renderSharesTable('reportTemplate', t('Report Templates'))}
+                {renderSharesTable('list', t('Lists'))}
+                {renderSharesTable('customForm', t('Custom Forms'))}
                 {renderSharesTable('report', t('Reports'))}
+                {renderSharesTable('reportTemplate', t('Report Templates'))}
             </div>
         );
     }

client/webpack.config.js

@@ -6,7 +6,8 @@ module.exports = {
         namespaces: ['babel-polyfill', './src/namespaces/root.js'],
         users: ['babel-polyfill', './src/users/root.js'],
         account: ['babel-polyfill', './src/account/root.js'],
-        reports: ['babel-polyfill', './src/reports/root.js']
+        reports: ['babel-polyfill', './src/reports/root.js'],
+        lists: ['babel-polyfill', './src/lists/root.js']
     },
     output: {
         library: 'MailtrainReactBody',

config/default.toml

@@ -200,31 +200,37 @@ permissions=["rebuildPermissions"]
 rootNamespaceRole="master"
 
 
-[roles.reportTemplate.master]
+[roles.namespace.master]
 name="Master"
 description="All permissions"
-permissions=["view", "edit", "delete", "share", "execute"]
+permissions=["view", "edit", "delete", "share", "createNamespace", "createList", "createCustomForm", "createReport", "createReportTemplate", "manageUsers"]
+
+[roles.namespace.master.children]
+list=["view", "edit", "delete"]
+customForm=["view", "edit", "delete"]
+report=["view", "edit", "delete", "share", "execute", "viewContent", "viewOutput"]
+reportTemplate=["view", "edit", "delete", "share", "execute"]
+namespace=["view", "edit", "delete", "share", "createNamespace", "createList", "createCustomForm", "createReport", "createReportTemplate", "manageUsers"]
+
+[roles.list.master]
+name="Master"
+description="All permissions"
+permissions=["view", "edit", "delete"]
+
+[roles.customForm.master]
+name="Master"
+description="All permissions"
+permissions=["view", "edit", "delete"]
 
 [roles.report.master]
 name="Master"
 description="All permissions"
 permissions=["view", "edit", "delete", "share", "execute", "viewContent", "viewOutput"]
 
-[roles.list.master]
+[roles.reportTemplate.master]
 name="Master"
 description="All permissions"
-permissions=[]
+permissions=["view", "edit", "delete", "share", "execute"]
-
-[roles.namespace.master]
-name="Master"
-description="All permissions"
-permissions=["view", "edit", "delete", "share", "createNamespace", "createReportTemplate", "createReport", "manageUsers"]
-
-[roles.namespace.master.children]
-reportTemplate=["view", "edit", "delete", "share", "execute"]
-report=["view", "edit", "delete", "share", "execute", "viewContent", "viewOutput"]
-list=[]
-namespace=["view", "edit", "delete", "share", "createNamespace", "createReportTemplate", "createReport", "manageUsers"]
 
 
 

lib/context-helpers.js

@@ -10,12 +10,13 @@ function getRequestContext(req) {
     return context;
 }
 
-function getServiceContext() {
+function getAdminContext() {
     const context = {
         user: {
-            id: 1,
+            admin: true,
-            username: 'admin',
+            id: 0,
-            name: 'Service worker',
+            username: '',
+            name: '',
             email: ''
         }
     };
@@ -25,5 +26,5 @@ function getServiceContext() {
 
 module.exports = {
     getRequestContext,
-    getServiceContext
+    getAdminContext
 };

lib/db.js

@@ -1,7 +1,7 @@
 'use strict';
 
 let config = require('config');
-let mysql = require('mysql');
+let mysql = require('mysql2');
 let redis = require('redis');
 let Lock = require('redfour');
 let stringifyDate = require('json-stringify-date');

lib/dbcheck.js

@@ -5,7 +5,7 @@ This module handles Mailtrain database initialization and upgrades
 */
 
 let config = require('config');
-let mysql = require('mysql');
+let mysql = require('mysql2');
 let log = require('npmlog');
 let fs = require('fs');
 let pathlib = require('path');

lib/models/lists.js

@@ -6,15 +6,7 @@ let shortid = require('shortid');
 let segments = require('./segments');
 let _ = require('../translate')._;
 let tableHelpers = require('../table-helpers');
-
+const UnsubscriptionMode = require('../../shared/lists').UnsubscriptionMode;
-const UnsubscriptionMode = {
-    ONE_STEP: 0,
-    ONE_STEP_WITH_FORM: 1,
-    TWO_STEP: 2,
-    TWO_STEP_WITH_FORM: 3,
-    MANUAL: 4,
-    MAX: 5
-};
 
 module.exports.UnsubscriptionMode = UnsubscriptionMode;
 

lib/namespace-helpers.js

@@ -11,6 +11,14 @@ async function validateEntity(tx, entity) {
     }
 }
 
+async function validateMove(context, entity, existing, entityTypeId, createOperation, deleteOperation) {
+    if (existing.namespace !== entity.namespace) {
+        await shares.enforceEntityPermission(context, 'namespace', entity.namespace, createOperation);
+        await shares.enforceEntityPermission(context, entityTypeId, entity.id, deleteOperation);
+    }
+}
+
 module.exports = {
-    validateEntity
+    validateEntity,
+    validateMove
 };

lib/passport.js

@@ -14,6 +14,7 @@ const bodyParser = require('body-parser');
 const users = require('../models/users');
 const { nodeifyFunction, nodeifyPromise } = require('./nodeify');
 const interoperableErrors = require('../shared/interoperable-errors');
+const contextHelpers = require('./context-helpers');
 
 let LdapStrategy;
 try {
@@ -143,6 +144,6 @@ if (config.ldap.enabled && LdapStrategy) {
     })));
 
     passport.serializeUser((user, done) => done(null, user.id));
-    passport.deserializeUser((id, done) => nodeifyPromise(users.getById(null, id), done));
+    passport.deserializeUser((id, done) => nodeifyPromise(users.getById(contextHelpers.getAdminContext(), id), done));
 }
 

lib/permissions.js

@@ -6,6 +6,16 @@ const entityTypes = {
         sharesTable: 'shares_namespace',
         permissionsTable: 'permissions_namespace'
     },
+    list: {
+        entitiesTable: 'lists',
+        sharesTable: 'shares_list',
+        permissionsTable: 'permissions_list'
+    },
+    customForm: {
+        entitiesTable: 'custom_forms',
+        sharesTable: 'shares_custom_form',
+        permissionsTable: 'permissions_custom_form'
+    },
     report: {
         entitiesTable: 'reports',
         sharesTable: 'shares_report',

models/forms.js

@@ -0,0 +1,261 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../shared/interoperable-errors');
+const shares = require('./shares');
+const bluebird = require('bluebird');
+const fs = bluebird.promisifyAll(require('fs'));
+const path = require('path');
+const mjml = require('mjml');
+const _ = require('../lib/translate')._;
+
+const formAllowedKeys = [
+    'name',
+    'description',
+    'layout',
+    'form_input_style'
+];
+
+const allowedFormKeys = [
+    'web_subscribe',
+    'web_confirm_subscription_notice',
+    'mail_confirm_subscription_html',
+    'mail_confirm_subscription_text',
+    'mail_already_subscribed_html',
+    'mail_already_subscribed_text',
+    'web_subscribed_notice',
+    'mail_subscription_confirmed_html',
+    'mail_subscription_confirmed_text',
+    'web_manage',
+    'web_manage_address',
+    'web_updated_notice',
+    'web_unsubscribe',
+    'web_confirm_unsubscription_notice',
+    'mail_confirm_unsubscription_html',
+    'mail_confirm_unsubscription_text',
+    'mail_confirm_address_change_html',
+    'mail_confirm_address_change_text',
+    'web_unsubscribed_notice',
+    'mail_unsubscription_confirmed_html',
+    'mail_unsubscription_confirmed_text',
+    'web_manual_unsubscribe_notice'
+];
+
+const hashKeys = [...formAllowedKeys, ...allowedFormKeys];
+
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, hashKeys));
+}
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'customForm', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('custom_forms')
+            .innerJoin('namespaces', 'namespaces.id', 'custom_forms.namespace'),
+        ['custom_forms.id', 'custom_forms.name', 'custom_forms.description', 'namespaces.name']
+    );
+}
+
+
+async function _getById(tx, id) {
+    const entity = await tx('custom_forms').where('id', id).first();
+
+    if (!entity) {
+        throw interoperableErrors.NotFoundError();
+    }
+
+    const forms = await tx('custom_forms_data').where('form', id).select(['data_key', 'data_value']);
+
+    for (const form of forms) {
+        entity[form.data_key] = form.data_value;
+    }
+
+    return entity;
+}
+
+
+async function getById(context, id) {
+    shares.enforceEntityPermission(context, 'customForm', id, 'view');
+
+    let entity;
+    await knex.transaction(async tx => {
+        entity = _getById(tx, id);
+    });
+
+    return entity;
+}
+
+
+async function serverValidate(context, data) {
+    const result = {};
+
+    const form = filterObject(data, allowedFormKeys);
+
+    const errs = checkForMjmlErrors(form);
+
+    for (const key in form) {
+        result[key] = {};
+        if (errs[key]) {
+            result.key.errors = errs[key];
+        }
+    }
+
+    return result;
+}
+
+
+async function create(context, entity) {
+    await shares.enforceEntityPermission(context, 'namespace', 'createCustomForm');
+
+    let id;
+    await knex.transaction(async tx => {
+        const ids = await tx('custom_forms').insert(filterObject(entity, formAllowedKeys));
+        id = ids[0];
+
+        const form = filterObject(entity, allowedFormKeys);
+        for (const formKey in form) {
+            await tx('custom_forms_data').insert({
+                form: id,
+                data_key: formKey,
+                data_value: form[formKey]
+            })
+        }
+    });
+
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await shares.enforceEntityPermission(context, 'customForm', entity.id, 'edit');
+
+    await knex.transaction(async tx => {
+        const existing = _getById(tx, context, id);
+
+        const existingHash = hash(existing);
+        if (existingHash != entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        const form = filterObject(entity, allowedFormKeys);
+        enforce(!Object.keys(checkForMjmlErrors(form)).length, 'Error(s) in form templates');
+
+        await tx('custom_forms').where('id', entity.id).update(filterObject(entity, formAllowedKeys));
+
+        for (const formKey in form) {
+            await tx('custom_forms_data').update({
+                form: entity.id,
+                data_key: formKey,
+                data_value: form[formKey]
+            });
+        }
+    });
+}
+
+async function remove(context, id) {
+    await knex.transaction(async tx => {
+        const entity = await tx('custom_forms').where('id', id).first();
+
+        if (!entity) {
+            throw shares.throwPermissionDenied();
+        }
+
+        shares.enforceEntityPermission(context, 'list', entity.list, 'manageForms');
+
+        await tx('custom_forms_data').where('form', id).del();
+        await tx('custom_forms').where('id', id).del();
+    });
+}
+
+
+async function getDefaultFormValues() {
+    const basePath = path.join(__dirname, '..');
+
+    async function getContents(fileName) {
+        try {
+            const template = await fs.readFile(path.join(basePath, fileName), 'utf8');
+            return template.replace(/\{\{#translate\}\}(.*?)\{\{\/translate\}\}/g, (m, s) => _(s));
+        } catch (err) {
+            return false;
+        }
+    }
+
+    const form = {};
+
+    for (const key of allowedFormKeys) {
+        const base = 'views/subscription/' + key.replace(/_/g, '-');
+        if (key.startsWith('mail') || key.startsWith('web')) {
+            form[key] = await getContents(base + '.mjml.hbs') || await getContents(base + '.hbs') || '';
+        }
+    }
+
+    form.layout = await getContents('views/subscription/layout.mjml.hbs') || '';
+    form.formInputStyle = await getContents('public/subscription/form-input-style.css') || '@import url(/subscription/form-input-style.css);';
+
+    return form;
+}
+
+
+function checkForMjmlErrors(form) {
+    let testLayout = '<mjml><mj-body><mj-container>{{{body}}}</mj-container></mj-body></mjml>';
+
+    let hasMjmlError = (template, layout = testLayout) => {
+        let source = layout.replace(/\{\{\{body\}\}\}/g, template);
+        let compiled;
+
+        try {
+            compiled = mjml.mjml2html(source);
+        } catch (err) {
+            return err;
+        }
+
+        return compiled.errors;
+    };
+
+
+    const errors = {};
+    for (const key in form) {
+        if (key.startsWith('mail_') || key.startsWith('web_')) {
+            const template = form[key];
+            const errs = hasMjmlError(template);
+
+            if (key === 'mail_confirm_html' && !template.includes('{{confirmUrl}}')) {
+                errs.push('Missing {{confirmUrl}}');
+            }
+
+            if (errs.length) {
+                errors[key] = errs;
+            }
+
+        } else if (key === 'layout') {
+            const layout = values[index];
+            const err = hasMjmlError('', layout);
+
+            if (!layout.includes('{{{body}}}')) {
+                errs.push(`{{{body}}} not found`);
+            }
+
+            if (errs.length) {
+                errors[key] = errs;
+            }
+        }
+    }
+
+    return errors;
+}
+
+module.exports = {
+    listDTAjax,
+    hash,
+    getById,
+    create,
+    updateWithConsistencyCheck,
+    remove,
+    getDefaultFormValues,
+    serverValidate
+};

models/lists.js

@@ -1,14 +1,38 @@
 'use strict';
 
 const knex = require('../lib/knex');
+const hasher = require('node-object-hash')();
 const dtHelpers = require('../lib/dt-helpers');
+const shortid = require('shortid');
+const { enforce, filterObject } = require('../lib/helpers');
 const interoperableErrors = require('../shared/interoperable-errors');
+const shares = require('./shares');
+const namespaceHelpers = require('../lib/namespace-helpers');
 
-async function listDTAjax(params) {
+const UnsubscriptionMode = require('../shared/lists').UnsubscriptionMode;
-    return await dtHelpers.ajaxList(params, builder => builder.from('lists'), ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description']);
+
+const allowedKeys = new Set(['name', 'description', 'default_form', 'public_subscribe', 'unsubscription_mode', 'namespace']);
+
+function hash(entity) {
+    return hasher.hash(filterObject(entity, allowedKeys));
 }
 
-async function getById(id) {
+
+async function listDTAjax(context, params) {
+    return await dtHelpers.ajaxListWithPermissions(
+        context,
+        [{ entityTypeId: 'list', requiredOperations: ['view'] }],
+        params,
+        builder => builder
+            .from('lists')
+            .innerJoin('namespaces', 'namespaces.id', 'lists.namespace'),
+        ['lists.id', 'lists.name', 'lists.cid', 'lists.subscribers', 'lists.description', 'namespaces.name']
+    );
+}
+
+async function getById(context, id) {
+    shares.enforceEntityPermission(context, 'list', id, 'view');
+
     const entity = await knex('lists').where('id', id).first();
     if (!entity) {
         throw new interoperableErrors.NotFoundError();
@@ -17,8 +41,68 @@ async function getById(id) {
     return entity;
 }
 
+async function create(context, entity) {
+    await shares.enforceEntityPermission(context, 'namespace', entity.namespace, 'createList');
+
+    let id;
+    await knex.transaction(async tx => {
+        await namespaceHelpers.validateEntity(tx, entity);
+        enforce(entity.unsubscription_mode >= 0 && entity.unsubscription_mode < UnsubscriptionMode.MAX, 'Unknown unsubscription mode');
+
+        const filteredEntity = filterObject(entity, allowedKeys);
+        filteredEntity.cid = shortid.generate();
+
+        const ids = await tx('lists').insert(filteredEntity);
+        id = ids[0];
+
+        await knex.schema.raw('CREATE TABLE `subscription__' + id + '` LIKE subscription');
+
+        await shares.rebuildPermissions(tx, { entityTypeId: 'list', entityId: id });
+    });
+
+    return id;
+}
+
+async function updateWithConsistencyCheck(context, entity) {
+    await shares.enforceEntityPermission(context, 'list', entity.id, 'edit');
+
+    await knex.transaction(async tx => {
+        const existing = await tx('lists').where('id', entity.id).first();
+        if (!existing) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        const existingHash = hash(existing);
+        if (existingHash != entity.originalHash) {
+            throw new interoperableErrors.ChangedError();
+        }
+
+        await namespaceHelpers.validateEntity(tx, entity);
+        await namespaceHelpers.validateMove(context, entity, existing, 'list', 'createList', 'delete');
+        enforce(entity.unsubscription_mode >= 0 && entity.unsubscription_mode < UnsubscriptionMode.MAX, 'Unknown unsubscription mode');
+
+        await tx('lists').where('id', entity.id).update(filterObject(entity, allowedKeys));
+
+        await shares.rebuildPermissions(tx, { entityTypeId: 'list', entityId: entity.id });
+    });
+}
+
+async function remove(context, id) {
+    await shares.enforceEntityPermission(context, 'list', id, 'delete');
+
+    await knex.transaction(async tx => {
+        await tx('lists').where('id', id).del();
+        await knex.schema.dropTableIfExists('subscription__' + id);
+    });
+}
+
 
 module.exports = {
+    UnsubscriptionMode,
+    hash,
     listDTAjax,
-    getById
+    getById,
+    create,
+    updateWithConsistencyCheck,
+    remove
 };

models/namespaces.js

@@ -151,6 +151,9 @@ async function updateWithConsistencyCheck(context, entity) {
             throw new interoperableErrors.ChangedError();
         }
 
+        // namespaceHelpers.validateEntity is not needed here because it is part of the tree traversal check below
+        await namespaceHelpers.validateMove(context, entity, existing, 'namespace', 'createNamespace', 'delete');
+
         let iter = entity;
         while (iter.namespace != null) {
             iter = await tx('namespaces').where('id', iter.namespace).first();

models/report-templates.js

@@ -66,11 +66,7 @@ async function updateWithConsistencyCheck(context, entity) {
         }
 
         await namespaceHelpers.validateEntity(tx, entity);
-
+        await namespaceHelpers.validateMove(context, entity, existing, 'reportTemplate', 'createReportTemplate', 'delete');
-        if (existing.namespace !== entity.namespace) {
-            await shares.enforceEntityPermission(context, 'namespace', entity.namespace, 'createReport');
-            await shares.enforceEntityPermission(context, 'reportTemplate', entity.id, 'delete');
-        }
 
         await tx('report_templates').where('id', entity.id).update(filterObject(entity, allowedKeys));
 

models/reports.js

@@ -19,9 +19,7 @@ function hash(entity) {
 }
 
 async function getByIdWithTemplate(context, id) {
-    if (context) {
     await shares.enforceEntityPermission(context, 'report', id, 'view');
-    }
 
     const entity = await knex('reports')
         .where('reports.id', id)
@@ -104,11 +102,7 @@ async function updateWithConsistencyCheck(context, entity) {
         }
 
         await namespaceHelpers.validateEntity(tx, entity);
-
+        await namespaceHelpers.validateMove(context, entity, existing, 'report', 'createReport', 'delete');
-        if (existing.namespace !== entity.namespace) {
-            await shares.enforceEntityPermission(context, 'namespace', entity.namespace, 'createReport');
-            await shares.enforceEntityPermission(context, 'report', entity.id, 'delete');
-        }
 
         entity.params = JSON.stringify(entity.params);
 

models/shares.js

@@ -22,7 +22,7 @@ async function listByEntityDTAjax(context, entityTypeId, entityId, params) {
             .innerJoin('generated_role_names', 'generated_role_names.role', 'users.role')
             .where('generated_role_names.entity_type', entityTypeId)
             .where(`${entityType.sharesTable}.entity`, entityId),
-        [ 'users.username', 'users.name', 'generated_role_names.name', 'users.id' ]
+        [ 'users.username', 'users.name', 'generated_role_names.name', 'users.id', entityType.sharesTable + '.auto' ]
     );
 }
 
@@ -46,7 +46,7 @@ async function listByUserDTAjax(context, entityTypeId, userId, params) {
             .innerJoin('generated_role_names', 'generated_role_names.role', entityType.sharesTable + '.role')
             .where('generated_role_names.entity_type', entityTypeId)
             .where(entityType.sharesTable + '.user', userId),
-        [ entityType.entitiesTable + '.name', 'generated_role_names.name', entityType.entitiesTable + '.id' ]
+        [ entityType.entitiesTable + '.name', 'generated_role_names.name', entityType.entitiesTable + '.id', entityType.sharesTable + '.auto' ]
     );
 }
 
@@ -167,7 +167,7 @@ async function _rebuildPermissions(tx, restriction) {
             const desiredRole = roleConf.ownNamespaceRole;
             if (desiredRole && user.role !== desiredRole) {
                 await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: user.namespace }).del();
-                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: user.namespace, role: desiredRole });
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: user.namespace, role: desiredRole, auto: true });
             }
         }
     }
@@ -191,7 +191,7 @@ async function _rebuildPermissions(tx, restriction) {
             const desiredRole = roleConf.rootNamespaceRole;
             if (desiredRole && user.role !== desiredRole) {
                 await tx(namespaceEntityType.sharesTable).where({ user: user.id, entity: 1 /* Global namespace id */ }).del();
-                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: 1 /* Global namespace id */, role: desiredRole });
+                await tx(namespaceEntityType.sharesTable).insert({ user: user.id, entity: 1 /* Global namespace id */, role: desiredRole, auto: 1 });
             }
         }
     }
@@ -268,7 +268,7 @@ async function _rebuildPermissions(tx, restriction) {
                         }
                     }
                 } else {
-                    const userPerms = {}
+                    const userPerms = {};
                     ns.transitiveUserPermissions.set(user, userPerms);
 
                     for (const entityTypeId in restrictedEntityTypes) {
@@ -407,6 +407,10 @@ async function removeDefaultShares(tx, user) {
 }
 
 function enforceGlobalPermission(context, requiredOperations) {
+    if (context.user.admin) { // This handles the getAdminContext() case
+        return;
+    }
+
     if (typeof requiredOperations === 'string') {
         requiredOperations = [ requiredOperations ];
     }
@@ -424,6 +428,10 @@ function enforceGlobalPermission(context, requiredOperations) {
 }
 
 async function _checkPermission(context, entityTypeId, entityId, requiredOperations) {
+    if (context.user.admin) { // This handles the getAdminContext() case
+        return true;
+    }
+
     const entityType = permissions.getEntityType(entityTypeId);
 
     if (typeof requiredOperations === 'string') {

models/users.js

@@ -31,7 +31,7 @@ const ownAccountAllowedKeys = new Set(['name', 'email', 'password']);
 const allowedKeysExternal = new Set(['username', 'namespace', 'role']);
 const hashKeys = new Set(['username', 'name', 'email', 'namespace', 'role']);
 const shares = require('./shares');
-
+const contextHelpers = require('../lib/context-helpers');
 
 function hash(entity) {
     return hasher.hash(filterObject(entity, hashKeys));
@@ -54,9 +54,7 @@ async function _getBy(context, key, value, extraColumns) {
         }
     }
 
-    if (context) {
     await shares.enforceEntityPermission(context, 'namespace', user.namespace, 'manageUsers');
-    }
 
     return user;
 }
@@ -260,16 +258,16 @@ async function remove(context, userId) {
 }
 
 async function getByAccessToken(accessToken) {
-    return await _getBy(null, 'access_token', accessToken);
+    return await _getBy(contextHelpers.getAdminContext(), 'access_token', accessToken);
 }
 
 async function getByUsername(username) {
-    return await _getBy(null, 'username', username);
+    return await _getBy(contextHelpers.getAdminContext(), 'username', username);
 }
 
 async function getByUsernameIfPasswordMatch(username, password) {
     try {
-        const user = await _getBy(null, 'username', username, ['password']);
+        const user = await _getBy(contextHelpers.getAdminContext(), 'username', username, ['password']);
 
         if (!await bcryptCompare(password, user.password)) {
             throw new interoperableErrors.IncorrectPasswordError();
@@ -287,7 +285,7 @@ async function getByUsernameIfPasswordMatch(username, password) {
 }
 
 async function getAccessToken(userId) {
-    const user = await _getBy(null, 'id', userId, ['access_token']);
+    const user = await _getBy(contextHelpers.getAdminContext(), 'id', userId, ['access_token']);
     return user.access_token;
 }
 

routes/lists-legacy-integration.js

@@ -0,0 +1,10 @@
+'use strict';
+
+const _ = require('../lib/translate')._;
+const clientHelpers = require('../lib/client-helpers');
+
+const router = require('../lib/router-async').create();
+
+clientHelpers.registerRootRoute(router, 'lists', _('Lists'));
+
+module.exports = router;

routes/lists-legacy.js

@@ -54,12 +54,15 @@ router.all('/*', (req, res, next) => {
     next();
 });
 
+/* REPLACED
 router.get('/', (req, res) => {
     res.render('lists/lists', {
         title: _('Lists')
     });
 });
+*/
 
+/* REPLACED
 router.get('/create', passport.csrfProtection, (req, res) => {
     let data = tools.convertKeys(req.query, {
         skip: ['layout']
@@ -75,7 +78,9 @@ router.get('/create', passport.csrfProtection, (req, res) => {
 
     res.render('lists/create', data);
 });
+*/
 
+/* REPLACED
 router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
     lists.create(req.body, (err, id) => {
         if (err || !id) {
@@ -86,7 +91,9 @@ router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) =
         res.redirect('/lists/view/' + id);
     });
 });
+*/
 
+/* REPLACED
 router.get('/edit/:id', passport.csrfProtection, (req, res) => {
     lists.get(req.params.id, (err, list) => {
         if (err || !list) {
@@ -112,7 +119,9 @@ router.get('/edit/:id', passport.csrfProtection, (req, res) => {
         });
     });
 });
+*/
 
+/* REPLACED
 router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
     lists.update(req.body.id, req.body, (err, updated) => {
 
@@ -133,7 +142,9 @@ router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) =>
         }
     });
 });
+*/
 
+/* REPLACED
 router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
     lists.delete(req.body.id, (err, deleted) => {
         if (err) {
@@ -147,6 +158,7 @@ router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) =
         return res.redirect('/lists');
     });
 });
+*/
 
 router.post('/ajax', (req, res) => {
     lists.filter(req.body, Number(req.query.parent) || false, (err, data, total, filteredTotal) => {

routes/reports.js

@@ -5,13 +5,14 @@ const _ = require('../lib/translate')._;
 const reports = require('../models/reports');
 const fileHelpers = require('../lib/file-helpers');
 const shares = require('../models/shares');
+const contextHelpers = require('../lib/context-helpers');
 
 const router = require('../lib/router-async').create();
 
 router.getAsync('/download/:id', passport.loggedIn, async (req, res) => {
     await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
 
-    const report = await reports.getByIdWithTemplate(null, req.params.id);
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
 
     if (report.state == reports.ReportState.FINISHED) {
         const headers = {

routes/rest/account.js

@@ -3,12 +3,13 @@
 const passport = require('../../lib/passport');
 const _ = require('../../lib/translate')._;
 const users = require('../../models/users');
+const contextHelpers = require('../../lib/context-helpers');
 
 const router = require('../../lib/router-async').create();
 
 
 router.getAsync('/account', passport.loggedIn, async (req, res) => {
-    const user = await users.getById(null, req.user.id);
+    const user = await users.getById(contextHelpers.getAdminContext(), req.user.id);
     user.hash = users.hash(user);
     return res.json(user);
 });

routes/rest/forms.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const forms = require('../../models/forms');
+
+const router = require('../../lib/router-async').create();
+
+
+router.postAsync('/forms-table', passport.loggedIn, async (req, res) => {
+    return res.json(await forms.listDTAjax(req.context, req.body));
+});
+
+router.getAsync('/forms/:formId', passport.loggedIn, async (req, res) => {
+    const entity = await forms.getById(req.context, req.params.formId);
+    entity.hash = forms.hash(entity);
+    return res.json(entity);
+});
+
+router.postAsync('/forms', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await forms.create(req.context, req.body);
+    return res.json();
+});
+
+router.putAsync('/forms/:formId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const entity = req.body;
+    entity.id = parseInt(req.params.formId);
+
+    await forms.updateWithConsistencyCheck(req.context, entity);
+    return res.json();
+});
+
+router.deleteAsync('/forms/:formId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await forms.remove(req.context, req.params.formId);
+    return res.json();
+});
+
+router.postAsync('/forms-validate', passport.loggedIn, async (req, res) => {
+    return res.json(await forms.serverValidate(req.context, req.body));
+});
+
+
+module.exports = router;

routes/rest/lists.js

@@ -7,7 +7,31 @@ const router = require('../../lib/router-async').create();
 
 
 router.postAsync('/lists-table', passport.loggedIn, async (req, res) => {
-    return res.json(await lists.listDTAjax(req.body));
+    return res.json(await lists.listDTAjax(req.context, req.body));
+});
+
+router.getAsync('/lists/:listId', passport.loggedIn, async (req, res) => {
+    const list = await lists.getById(req.context, req.params.listId);
+    list.hash = lists.hash(list);
+    return res.json(list);
+});
+
+router.postAsync('/lists', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await lists.create(req.context, req.body);
+    return res.json();
+});
+
+router.putAsync('/lists/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    const list = req.body;
+    list.id = parseInt(req.params.listId);
+
+    await lists.updateWithConsistencyCheck(req.context, list);
+    return res.json();
+});
+
+router.deleteAsync('/lists/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    await lists.remove(req.context, req.params.listId);
+    return res.json();
 });
 
 

routes/rest/reports.js

@@ -6,6 +6,7 @@ const reports = require('../../models/reports');
 const reportProcessor = require('../../lib/report-processor');
 const fileHelpers = require('../../lib/file-helpers');
 const shares = require('../../models/shares');
+const contextHelpers = require('../../lib/context-helpers');
 
 const router = require('../../lib/router-async').create();
 
@@ -41,7 +42,7 @@ router.postAsync('/reports-table', passport.loggedIn, async (req, res) => {
 router.postAsync('/report-start/:id', passport.loggedIn, passport.csrfProtection, async (req, res) => {
     await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'execute');
 
-    const report = await reports.getByIdWithTemplate(null, req.params.id);
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
     await shares.enforceEntityPermission(req.context, 'reportTemplate', report.report_template, 'execute');
 
     await reportProcessor.start(req.params.id);
@@ -51,7 +52,7 @@ router.postAsync('/report-start/:id', passport.loggedIn, passport.csrfProtection
 router.postAsync('/report-stop/:id', async (req, res) => {
     await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'execute');
 
-    const report = await reports.getByIdWithTemplate(null, req.params.id);
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
     await shares.enforceEntityPermission(req.context, 'reportTemplate', report.report_template, 'execute');
 
     await reportProcessor.stop(req.params.id);
@@ -61,14 +62,14 @@ router.postAsync('/report-stop/:id', async (req, res) => {
 router.getAsync('/report-content/:id', async (req, res) => {
     await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewContent');
 
-    const report = await reports.getByIdWithTemplate(null, req.params.id);
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
     res.sendFile(fileHelpers.getReportContentFile(report));
 });
 
 router.getAsync('/report-output/:id', async (req, res) => {
     await shares.enforceEntityPermission(req.context, 'report', req.params.id, 'viewOutput');
 
-    const report = await reports.getByIdWithTemplate(null, req.params.id);
+    const report = await reports.getByIdWithTemplate(contextHelpers.getAdminContext(), req.params.id);
     res.sendFile(fileHelpers.getReportOutputFile(report));
 });
 

setup/knex/migrations/20170507083345_create_namespaces.js

@@ -1,8 +1,6 @@
 exports.up = function(knex, Promise) {
-    const entityTypesAddNamespace = ['list', 'report', 'report_template', 'user'];
+    const entityTypesAddNamespace = ['list', 'custom_form', 'report', 'report_template', 'user'];
-    let schema = knex.schema;
+    let promise = knex.schema.createTable('namespaces', table => {
-
-    schema = schema.createTable('namespaces', table => {
             table.increments('id').primary();
             table.string('name');
             table.text('description');
@@ -10,12 +8,12 @@ exports.up = function(knex, Promise) {
         })
         .then(() => knex('namespaces').insert({
             id: 1, /* Global namespace id */
-            name: 'Global',
+            name: 'Root',
-            description: 'Global namespace'
+            description: 'Root namespace'
         }));
 
     for (const entityType of entityTypesAddNamespace) {
-        schema = schema
+        promise = promise
             .then(() => knex.schema.table(`${entityType}s`, table => {
                 table.integer('namespace').unsigned().notNullable();
             }))
@@ -27,7 +25,7 @@ exports.up = function(knex, Promise) {
             }));
     }
 
-    return schema;
+    return promise;
 };
 
 exports.down = function(knex, Promise) {

setup/knex/migrations/20170507084114_create_permissions.js

@@ -1,4 +1,4 @@
-const shareableEntityTypes = ['list', 'report', 'report_template', 'namespace'];
+const shareableEntityTypes = ['list', 'custom_form', 'report', 'report_template', 'namespace'];
 
 exports.up = function(knex, Promise) {
     let schema = knex.schema;
@@ -9,6 +9,7 @@ exports.up = function(knex, Promise) {
                 table.integer('entity').unsigned().notNullable().references(`${entityType}s.id`).onDelete('CASCADE');
                 table.integer('user').unsigned().notNullable().references('users.id').onDelete('CASCADE');
                 table.string('role', 128).notNullable();
+                table.boolean('auto').defaultTo(false);
                 table.primary(['entity', 'user']);
             })
             .createTable(`permissions_${entityType}`, table => {

setup/knex/migrations/20170728220422_drop_id_in_custom_forms_data.js

@@ -0,0 +1,10 @@
+exports.up = function(knex, Promise) {
+    return knex.schema.table('custom_forms_data', table => {
+        table.dropColumn('id');
+        table.string('data_key', 128).alter();
+        table.primary(['form', 'data_key']);
+    })
+};
+
+exports.down = function(knex, Promise) {
+};

setup/knex/migrations/20170729160135_make_custom_forms_independent_of_list.js

@@ -0,0 +1,9 @@
+exports.up = function(knex, Promise) {
+    return knex.schema.table('custom_forms', table => {
+        table.dropForeign('list', 'custom_forms_ibfk_1');
+        table.dropColumn('list');
+    })
+};
+
+exports.down = function(knex, Promise) {
+};

setup/knex/migrations/20170729160422_move_form_field_order_to_custom_fields_and_make_all_fields_configurable.js

@@ -0,0 +1,97 @@
+"use strict";
+
+exports.up = (knex, Promise) => (async() => {
+    await knex.schema.table('custom_fields', table => {
+        table.integer('order_subscribe');
+        table.integer('order_manage');
+        table.integer('order_list');
+    });
+
+    await knex.schema.table('subscription', table => {
+        table.dropColumn('first_name');
+        table.dropColumn('last_name');
+    });
+
+    const lists = await knex('lists')
+        .leftJoin('custom_forms', 'lists.default_form', 'custom_forms.id')
+        .select(['lists.id', 'lists.default_form', 'custom_forms.fields_shown_on_subscribe', 'custom_forms.fields_shown_on_manage']);
+
+    for (const list of lists) {
+        const fields = await knex('custom_fields').where('list', list.id).orderBy('id', 'asc');
+
+        const [firstNameFieldId] = await knex('custom_fields').insert({
+            list: list.id,
+            name: 'First Name',
+            key: 'FIRST_NAME',
+            type: 'text',
+            column: 'first_name',
+            visible: 1 // FIXME - Revise the need for this field
+        });
+
+        const [lastNameFieldId] = await knex('custom_fields').insert({
+            list: list.id,
+            name: 'Last Name',
+            key: 'LAST_NAME',
+            type: 'text',
+            column: 'last_name',
+            visible: 1 // FIXME - Revise the need for this field
+        });
+
+        let orderSubscribe;
+        let orderManage;
+
+        const replaceNames = x => {
+            if (x === 'firstname') {
+                return firstNameFieldId;
+            } else if (x === 'lastname') {
+                return lastNameFieldId;
+            } else {
+                return x;
+            }
+        };
+
+        if (list.default_form) {
+            orderSubscribe = list.fields_shown_on_subscribe.split(',').map(replaceNames);
+            orderManage = list.fields_shown_on_subscribe.split(',').map(replaceNames);
+        } else {
+            orderSubscribe = [firstNameFieldId, lastNameFieldId];
+            orderManage = [firstNameFieldId, lastNameFieldId];
+
+            for (const fld of fields) {
+                if (fld.visible && fld.type !== 'option') {
+                    orderSubscribe.push(fld.id);
+                    orderManage.push(fld.id);
+                }
+            }
+        }
+
+        const orderList = [firstNameFieldId, lastNameFieldId];
+
+        let idx = 0;
+        for (const fldId of orderSubscribe) {
+            await knex('custom_fields').where('id', fldId).update({order_subscribe: idx});
+            idx += 1;
+        }
+
+        idx = 0;
+        for (const fldId of orderManage) {
+            await knex('custom_fields').where('id', fldId).update({order_manage: idx});
+            idx += 1;
+        }
+
+        idx = 0;
+        for (const fldId of orderList) {
+            await knex('custom_fields').where('id', fldId).update({order_list: idx});
+            idx += 1;
+        }
+    }
+
+    await knex.schema.table('custom_forms', table => {
+        table.dropColumn('fields_shown_on_subscribe');
+        table.dropColumn('fields_shown_on_manage');
+    });
+})();
+
+
+exports.down = function(knex, Promise) {
+};

shared/lists.js

@@ -0,0 +1,14 @@
+'use strict';
+
+const UnsubscriptionMode = {
+    ONE_STEP: 0,
+    ONE_STEP_WITH_FORM: 1,
+    TWO_STEP: 2,
+    TWO_STEP_WITH_FORM: 3,
+    MANUAL: 4,
+    MAX: 5
+};
+
+module.exports = {
+    UnsubscriptionMode
+};

workers/reports/report-processor.js

@@ -18,15 +18,14 @@ const contextHelpers = require('../../lib/context-helpers');
 
 handlebarsHelpers.registerHelpers(handlebars);
 
-const userFieldGetters = {
-    'campaign': campaigns.getById,
-    'list': lists.getById
-};
-
-
 async function main() {
     try {
-        const context = contextHelpers.getServiceContext();
+        const context = contextHelpers.getAdminContext();
+
+        const userFieldGetters = {
+            'campaign': campaigns.getById,
+            'list': id => lists.getById(context, id)
+        };
 
         const reportId = Number(process.argv[2]);