Implemented basic support for GDPR

client/src/lists/forms/CUD.js

@@ -69,7 +69,8 @@ export default class CUD extends Component {
             'web_unsubscribed_notice',
             'mail_unsubscription_confirmed_html',
             'mail_unsubscription_confirmed_text',
-            'web_manual_unsubscribe_notice'
+            'web_manual_unsubscribe_notice',
+            'web_privacy_policy_notice'
         ];
 
         this.initForm({
@@ -206,6 +207,11 @@ export default class CUD extends Component {
                 label: t('webManualUnsubscribeNotice'),
                 mode: 'html',
                 help: helpMjmlGeneral
+            },
+            web_privacy_policy_notice: {
+                label: t('Privacy policy'),
+                mode: 'html',
+                help: helpMjmlGeneral
             }
         };
 
@@ -254,6 +260,12 @@ export default class CUD extends Component {
                     'web_manual_unsubscribe_notice'
                 ]
             },
+            gdpr: {
+                label: t('Data protection'),
+                options: [
+                    'web_privacy_policy_notice'
+                ]
+            },
         };
 
     }

client/src/lists/imports/CUD.js

@@ -39,6 +39,8 @@ import axios from "../../lib/axios";
 import {getUrl} from "../../lib/urls";
 import listStyles from "../styles.scss";
 import styles from "../../lib/styles.scss";
+import interoperableErrors
+    from "../../../../shared/interoperable-errors";
 
 
 function truncate(str, len, ending = '...') {
@@ -185,6 +187,10 @@ export default class CUD extends Component {
     }
 
     async submitHandler() {
+        await this.save();
+    }
+
+    async save(runAfterSave) {
         const t = this.props.t;
         const isEdit = !!this.props.entity;
 
@@ -275,6 +281,16 @@ export default class CUD extends Component {
                 if (!isEdit) {
                     this.navigateTo(`/lists/${this.props.list.id}/imports/${submitResponse}/edit`);
                 } else {
+                    try {
+                        await axios.post(getUrl(`rest/import-start/${this.props.list.id}/${this.props.entity.id}`));
+                    } catch (err) {
+                        if (err instanceof interoperableErrors.InvalidStateError) {
+                            // Just mask the fact that it's not possible to start anything and refresh instead.
+                        } else {
+                            throw err;
+                        }
+                    }
+
                     this.navigateToWithFlashMessage(`/lists/${this.props.list.id}/imports/${this.props.entity.id}/status`, 'success', t('importSaved'));
                 }
 
@@ -389,11 +405,12 @@ export default class CUD extends Component {
             }
         }
 
-        let saveButtonLabel;
+        const saveButtons = []
         if (!isEdit) {
-            saveButtonLabel = t('saveAndEditSettings');
+            saveButtons.push(<Button key="default" type="submit" className="btn-primary" icon="ok" label={t('saveAndEditSettings')}/>);
         } else {
-            saveButtonLabel = t('save');
+            saveButtons.push(<Button key="default" type="submit" className="btn-primary" icon="ok" label={t('save')}/>);
+            saveButtons.push(<Button key="saveAndRun" className="btn-primary" icon="ok" label={t('Save and Run')} onClickAsync={async () => await this.save(true)}/>);
         }
 
         return (
@@ -427,7 +444,7 @@ export default class CUD extends Component {
 
 
                     <ButtonRow>
-                        <Button type="submit" className="btn-primary" icon="ok" label={saveButtonLabel}/>
+                        {saveButtons}
                         {isEdit && <NavButton className="btn-danger" icon="remove" label={t('delete')} linkTo={`/lists/${this.props.list.id}/imports/${this.props.entity.id}/delete`}/>}
                     </ButtonRow>
                 </Form>

client/src/lists/segments/helpers.js

@@ -96,13 +96,13 @@ export function getRuleHelpers(t, fields) {
     };
 
     // FXIME - the localization here is still wrong
-    function getRelativeDateTreeLabel(rule, textFragment) {
+    function getRelativeDateTreeLabel(rule, variants) {
         if (rule.value === 0) {
-            return t(/*ignore*/'Date in column ' + textFragment + ' the current date', {colName: ruleHelpers.getColumnName(rule.column)})
+            return t(variants[0], {colName: ruleHelpers.getColumnName(rule.column)})
         } else if (rule.value > 0) {
-            return t(/*ignore*/'Date in column ' + textFragment + ' {{value}}-th day after the current date', {colName: ruleHelpers.getColumnName(rule.column), value: rule.value});
+            return t(variants[1], {colName: ruleHelpers.getColumnName(rule.column), value: rule.value});
         } else {
-            return t(/*ignore*/'Date in column ' + textFragment + ' {{value}}-th day before the current date', {colName: ruleHelpers.getColumnName(rule.column), value: rule.value});
+            return t(variants[2], {colName: ruleHelpers.getColumnName(rule.column), value: rule.value});
         }
     }
 
@@ -129,48 +129,23 @@ export function getRuleHelpers(t, fields) {
         },
         eqTodayPlusDays: {
             dropdownLabel: t('onXthDayBeforeafterCurrentDate'),
-            /*
+            treeLabel: rule => getRelativeDateTreeLabel(rule, 'is', [tMark('dateInColumnIsTheCurrentDate'), tMark('dateInColumnIsValuethDayAfterTheCurrent'), tMark('dateInColumnIsValuethDayBeforeTheCurrent')]),
-            tMark('dateInColumnIsTheCurrentDate')
-            tMark('dateInColumnIsValuethDayAfterTheCurrent')
-            tMark('dateInColumnIsValuethDayBeforeTheCurrent')
-             */
-            treeLabel: rule => getRelativeDateTreeLabel(rule, 'is'),
         },
         ltTodayPlusDays: {
             dropdownLabel: t('beforeXthDayBeforeafterCurrentDate'),
-            /*
+            treeLabel: rule => getRelativeDateTreeLabel(rule, [tMark('dateInColumnIsBeforeTheCurrentDate'), tMark('dateInColumnIsBeforeValuethDayAfterThe'), tMark('dateInColumnIsBeforeValuethDayBeforeThe')]),
-            tMark('dateInColumnIsBeforeTheCurrentDate')
-            tMark('dateInColumnIsBeforeValuethDayAfterThe')
-            tMark('dateInColumnIsBeforeValuethDayBeforeThe')
-             */
-            treeLabel: rule => getRelativeDateTreeLabel(rule, 'is before'),
         },
         leTodayPlusDays: {
             dropdownLabel: t('beforeOrOnXthDayBeforeafterCurrentDate'),
-            /*
+            treeLabel: rule => getRelativeDateTreeLabel(rule, [tMark('dateInColumnIsBeforeOrOnTheCurrentDate'), tMark('dateInColumnIsBeforeOrOnValuethDayAfter'), tMark('dateInColumnIsBeforeOrOnValuethDayBefore')]),
-            tMark('dateInColumnIsBeforeOrOnTheCurrentDate')
-            tMark('dateInColumnIsBeforeOrOnValuethDayAfter')
-            tMark('dateInColumnIsBeforeOrOnValuethDayBefore')
-             */
-            treeLabel: rule => getRelativeDateTreeLabel(rule, 'is before or on'),
         },
         gtTodayPlusDays: {
             dropdownLabel: t('afterXthDayBeforeafterCurrentDate'),
-            /*
+            treeLabel: rule => getRelativeDateTreeLabel(rule, [tMark('dateInColumnIsAfterTheCurrentDate'), tMark('dateInColumnIsAfterValuethDayAfterThe'), tMark('dateInColumnIsAfterValuethDayAfterThe')]),
-            tMark('dateInColumnIsAfterTheCurrentDate')
-            tMark('dateInColumnIsAfterValuethDayAfterThe')
-            tMark('dateInColumnIsAfterValuethDayAfterThe')
-             */
-            treeLabel: rule => getRelativeDateTreeLabel(rule, 'is after'),
         },
         geTodayPlusDays: {
             dropdownLabel: t('afterOrOnXthDayBeforeafterCurrentDate'),
-            /*
+            treeLabel: rule => getRelativeDateTreeLabel(rule, [tMark('dateInColumnIsAfterOrOnTheCurrentDate'), tMark('dateInColumnIsAfterOrOnValuethDayAfter'), tMark('dateInColumnIsAfterOrOnValuethDayAfter')]),
-            tMark('dateInColumnIsAfterOrOnTheCurrentDate')
-            tMark('dateInColumnIsAfterOrOnValuethDayAfter')
-            tMark('dateInColumnIsAfterOrOnValuethDayAfter')
-             */
-            treeLabel: rule => getRelativeDateTreeLabel(rule, 'is after or on'),
         }
     };
 

client/src/lists/styles.scss

@@ -1,3 +1,7 @@
 .mapping {
   margin-top: 30px;
+}
+
+.erased {
+  color: #808080;
 }

client/src/lists/subscriptions/List.js

@@ -13,14 +13,18 @@ import {
     withForm
 } from '../../lib/form';
 import {Icon, Button} from "../../lib/bootstrap-components";
-import axios from '../../lib/axios';
+import axios, {HTTPMethod} from '../../lib/axios';
 import {getFieldTypes, getSubscriptionStatusLabels} from './helpers';
 import {getUrl, getPublicUrl} from "../../lib/urls";
 import {
+    DeleteModalDialog,
+    RestActionModalDialog,
     tableDeleteDialogAddDeleteButton,
     tableDeleteDialogInit,
     tableDeleteDialogRender
 } from "../../lib/modals";
+import listStyles from "../styles.scss";
+import styles from '../../lib/styles.scss';
 
 @withTranslation()
 @withForm
@@ -87,7 +91,7 @@ export default class List extends Component {
 
         const columns = [
             { data: 1, title: t('id'), render: data => <code>{data}</code> },
-            { data: 2, title: t('email') },
+            { data: 2, title: t('email'), render: data => data === null ? <span className={listStyles.erased}>{t('[ERASED]')}</span> : data },
             { data: 3, title: t('status'), render: (data, display, rowData) => this.subscriptionStatusLabels[data] + (rowData[5] ? ', ' + t('blacklisted') : '') },
             { data: 4, title: t('created'), render: data => data ? moment(data).fromNow() : '' }
         ];
@@ -112,27 +116,30 @@ export default class List extends Component {
             columns.push({
                 actions: data => {
                     const actions = [];
+                    const id = data[0];
+                    const email = data[2];
+                    const status = data[3];
 
                     actions.push({
                         label: <Icon icon="edit" title={t('edit')}/>,
-                        link: `/lists/${this.props.list.id}/subscriptions/${data[0]}/edit`
+                        link: `/lists/${this.props.list.id}/subscriptions/${id}/edit`
                     });
 
-                    if (data[3] === SubscriptionStatus.SUBSCRIBED) {
+                    if (email && status === SubscriptionStatus.SUBSCRIBED) {
                         actions.push({
                             label: <Icon icon="off" title={t('unsubscribe')}/>,
-                            action: () => this.unsubscribeSubscription(data[0])
+                            action: () => this.unsubscribeSubscription(id)
                         });
                     }
 
-                    if (!data[5]) {
+                    if (email && !data[5]) {
                         actions.push({
                             label: <Icon icon="ban-circle" title={t('blacklist')}/>,
-                            action: () => this.blacklistSubscription(data[2])
+                            action: () => this.blacklistSubscription(email)
                         });
                     }
 
-                    tableDeleteDialogAddDeleteButton(actions, this, null, data[0], data[2]);
+                    tableDeleteDialogAddDeleteButton(actions, this, null, id, email);
 
                     return actions;
                 }

client/src/root.js

@@ -49,7 +49,11 @@ import axios
 import {getUrl} from "./lib/urls";
 import {langCodes} from "../../shared/langs";
 
-const topLevelMenuKeys = ['lists', 'templates', 'campaigns', 'reports'];
+const topLevelMenuKeys = ['lists', 'templates', 'campaigns'];
+
+if (mailtrainConfig.reportsEnabmed) {
+    topLevelMenuKeys.push('reports');
+}
 
 @withTranslation()
 class Root extends Component {

server/config/default.yaml

@@ -14,6 +14,20 @@
 # Process title visible in monitoring logs and process listing
 title: mailtrain
 
+# GDPR related settings (https://en.wikipedia.org/wiki/General_Data_Protection_Regulation)
+gdpr:
+  # If enabled, after a user unsubscribes or complains, his/her data are removed from the subscription.
+  # Only hashed email remains to prevent resubscription via import
+  deleteDataAfterUnsubscribe:
+    enabled: true
+    secondsAfterUnsubscribe: 86400 # 1 day
+
+  # If enabled, after the time below, an entry of a subscriber that unsubscribed or complained
+  # is completely removed from the subscribers list (including the hashed email)
+  deleteSubscriptionAfterUnsubscribe:
+    enabled: true
+    secondsAfterUnsubscribe: 2592000 # 30 days
+
 # Enabled HTML editors
 editors:
 - grapesjs
@@ -178,7 +192,7 @@ reports:
   # performing network operations and in generating XSS attacks as part of the report.
   # The bottom line is that if people who are creating report templates or have write access to the DB cannot be trusted,
   # then it's safer to switch off the reporting functionality below.
-  enabled: false
+  enabled: true
 
 testServer:
   # Starts a vanity server that redirects all mail to /dev/null

server/index.js

@@ -6,6 +6,7 @@ const appBuilder = require('./app-builder');
 const translate = require('./lib/translate');
 const http = require('http');
 const triggers = require('./services/triggers');
+const gdprCleanup = require('./services/gdpr-cleanup');
 const importer = require('./lib/importer');
 const feedcheck = require('./lib/feedcheck');
 const verpServer = require('./services/verp-server');
@@ -105,6 +106,7 @@ dbcheck(err => { // Check if database needs upgrading before starting the server
                                     feedcheck.spawn(() => {
                                         senders.spawn(() => {
                                             triggers.start();
+                                            gdprCleanup.start();
 
                                             postfixBounceServer(async () => {
                                                 (async () => {

server/lib/client-helpers.js

@@ -41,7 +41,8 @@ async function getAuthenticatedConfig(context) {
         globalPermissions,
         editors: config.editors,
         mosaico: config.mosaico,
-        verpEnabled: config.verp.enabled
+        verpEnabled: config.verp.enabled,
+        reportsEnabled: config.reports.enabled
     }
 }
 

server/models/blacklist.js

@@ -34,6 +34,8 @@ async function search(context, offset, limit, search) {
 }
 
 async function add(context, email) {
+    enforce(email, 'Email has to be set');
+
     return await knex.transaction(async tx => {
         shares.enforceGlobalPermission(context, 'manageBlacklist');
 
@@ -45,11 +47,15 @@ async function add(context, email) {
 }
 
 async function remove(context, email) {
+    enforce(email, 'Email has to be set');
+
     shares.enforceGlobalPermission(context, 'manageBlacklist');
     await knex('blacklist').where('email', email).del();
 }
 
 async function isBlacklisted(email) {
+    enforce(email, 'Email has to be set');
+
     const existing = await knex('blacklist').where('email', email).first();
     return !!existing;
 }

server/models/forms.js

@@ -9,7 +9,10 @@ const shares = require('./shares');
 const namespaceHelpers = require('../lib/namespace-helpers');
 const fs = require('fs-extra');
 const path = require('path');
+
 const mjml = require('mjml');
+const mjml2html = mjml.default;
+
 const lists = require('./lists');
 const dependencyHelpers = require('../lib/dependency-helpers');
 
@@ -43,7 +46,8 @@ const allowedFormKeys = new Set([
     'web_unsubscribed_notice',
     'mail_unsubscription_confirmed_html',
     'mail_unsubscription_confirmed_text',
-    'web_manual_unsubscribe_notice'
+    'web_manual_unsubscribe_notice',
+    'web_privacy_policy_notice'
 ]);
 
 const hashKeys = new Set([...formAllowedKeys, ...allowedFormKeys]);
@@ -84,11 +88,15 @@ async function _getById(tx, id) {
 }
 
 
-async function getById(context, id) {
+async function getById(context, id, withPermissions = true) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'customForm', id, 'view');
         const entity = await _getById(tx, id);
-        entity.permissions = await shares.getPermissionsTx(tx, context, 'customForm', id);
+
+        if (withPermissions) {
+            entity.permissions = await shares.getPermissionsTx(tx, context, 'customForm', id);
+        }
+
         return entity;
     });
 }
@@ -219,7 +227,7 @@ function checkForMjmlErrors(form) {
         let compiled;
 
         try {
-            compiled = mjml(source);
+            compiled = mjml2html(source);
         } catch (err) {
             return err;
         }
@@ -230,7 +238,7 @@ function checkForMjmlErrors(form) {
 
     const errors = {};
     for (const key in form) {
-        if (key.startsWith('mail_') || key.startsWith('web_')) {
+        if ((key.startsWith('mail_') && key.endsWith('_html')) || key.startsWith('web_')) {
             const template = form[key];
             const errs = hasMjmlError(template);
 

server/models/lists.js

@@ -121,7 +121,7 @@ async function create(context, entity) {
         await knex.schema.raw('CREATE TABLE `subscription__' + id + '` (\n' +
             '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
             '  `cid` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
-            '  `email` varchar(255) CHARACTER SET utf8 NOT NULL,\n' +
+            '  `email` varchar(255) CHARACTER SET utf8 DEFAULT NULL,\n' +
             '  `hash_email` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
             '  `source_email` int(10) unsigned,\n' + // This references imports if the source is an import, 0 means some import in version 1, NULL if the source is via subscription or edit of the subscription
             '  `opt_in_ip` varchar(100) DEFAULT NULL,\n' +
@@ -134,8 +134,9 @@ async function create(context, entity) {
             '  `latest_click` timestamp NULL DEFAULT NULL,\n' +
             '  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
             '  PRIMARY KEY (`id`),\n' +
-            '  UNIQUE KEY `email` (`email`),\n' +
+            '  UNIQUE KEY `hash_email` (`hash_email`),\n' +
             '  UNIQUE KEY `cid` (`cid`),\n' +
+            '  KEY `email` (`email`),\n' +
             '  KEY `status` (`status`),\n' +
             '  KEY `subscriber_tz` (`tz`),\n' +
             '  KEY `is_test` (`is_test`),\n' +

server/models/subscriptions.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const config = require('config');
 const knex = require('../lib/knex');
 const hasher = require('node-object-hash')();
 const shortid = require('shortid');
@@ -7,7 +8,7 @@ const dtHelpers = require('../lib/dt-helpers');
 const interoperableErrors = require('../../shared/interoperable-errors');
 const shares = require('./shares');
 const fields = require('./fields');
-const { SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
+const { SubscriptionSource, SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
 const segments = require('./segments');
 const { enforce, filterObject } = require('../lib/helpers');
 const moment = require('moment');
@@ -475,7 +476,7 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
     }
     const existingWithKey = await existingWithKeyQuery.first();
     if (existingWithKey) {
-        if (meta && (meta.updateAllowed || (meta.updateOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED))) {
+        if (existingWithKey.email === null || meta.updateAllowed || (meta.updateOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED)) {
             meta.update = true;
             meta.existing = existingWithKey;
         } else {
@@ -486,12 +487,12 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
         // The same for import where we need to subscribed only those (existing and new) that have not been unsubscribed already.
         // In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status to SUBSCRIBED
 
-        if (meta && meta.subscribeIfNoExisting && !entity.status) {
+        if (meta.subscribeIfNoExisting && !entity.status) {
             entity.status = SubscriptionStatus.SUBSCRIBED;
         }
     }
 
-    if ((isCreate && !(meta && meta.update)) || 'status' in entity) {
+    if ((isCreate && !meta.update) || 'status' in entity) {
         enforce(entity.status >= SubscriptionStatus.MIN && entity.status <= SubscriptionStatus.MAX, 'Invalid status');
     }
 
@@ -532,26 +533,69 @@ function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
     }
 }
 
-async function _update(tx, listId, existing, filteredEntity) {
+
+
+function purgeSensitiveData(subscription, groupedFieldsMap) {
+    subscription.email = null;
+
+    for (const fldCol in groupedFieldsMap) {
+        const fld = groupedFieldsMap[fldCol];
+
+        const fieldType = fields.getFieldType(fld.type);
+        if (fieldType.grouped) {
+            for (const optionKey in fld.groupedOptions) {
+                const option = fld.groupedOptions[optionKey];
+                subscription[option.column] = null;
+                subscription['source_' + option.column] = SubscriptionSource.ERASED;
+            }
+        } else {
+            subscription[fldCol] = null;
+            subscription['source_' + fldCol] = SubscriptionSource.ERASED;
+        }
+    }
+}
+
+async function _update(tx, listId, groupedFieldsMap, existing, filteredEntity) {
     if ('status' in filteredEntity) {
         if (existing.status !== filteredEntity.status) {
             filteredEntity.status_change = new Date();
         }
     }
 
-    console.log(filteredEntity);
+    if (filteredEntity.status === SubscriptionStatus.UNSUBSCRIBED || filteredEntity.status === SubscriptionStatus.COMPLAINED) {
-    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
+        if (existing.unsubscribed === null) {
-
+            filteredEntity.unsubscribed = new Date();
-    if ('status' in filteredEntity) {
-        let countIncrement = 0;
-        if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
-            countIncrement = -1;
-        } else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
-            countIncrement = 1;
         }
 
-        if (countIncrement) {
+        if (config.gdpr.deleteSubscriptionAfterUnsubscribe.enabled && config.gdpr.deleteSubscriptionAfterUnsubscribe.secondsAfterUnsubscribe === 0) {
-            await tx('lists').where('id', listId).increment('subscribers', countIncrement);
+            filteredEntity = null;
+        } else if (config.gdpr.deleteDataAfterUnsubscribe.enabled && config.gdpr.deleteDataAfterUnsubscribe.secondsAfterUnsubscribe === 0) {
+            purgeSensitiveData(filteredEntity, groupedFieldsMap);
+        }
+    } else if (filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
+        filteredEntity.unsubscribed = null;
+    }
+
+    if (filteredEntity) {
+        await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
+
+        if ('status' in filteredEntity) {
+            let countIncrement = 0;
+            if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
+                countIncrement = -1;
+            } else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
+                countIncrement = 1;
+            }
+
+            if (countIncrement) {
+                await tx('lists').where('id', listId).increment('subscribers', countIncrement);
+            }
+        }
+    } else {
+        await tx(getSubscriptionTableName(listId)).where('id', existing.id).del();
+
+        if (existing.status === SubscriptionStatus.SUBSCRIBED) {
+            await tx('lists').where('id', listId).decrement('subscribers', 1);
         }
     }
 }
@@ -586,20 +630,17 @@ async function createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMa
 
     updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
 
-    filteredEntity.opt_in_ip = meta && meta.ip;
+    filteredEntity.opt_in_ip = meta.ip;
-    filteredEntity.opt_in_country = meta && meta.country;
+    filteredEntity.opt_in_country = meta.country;
 
-    if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
+    if (meta.update) { // meta.update is set by _validateAndPreprocess
-        await _update(tx, listId, meta.existing, filteredEntity);
+        await _update(tx, listId, groupedFieldsMap, meta.existing, filteredEntity);
         meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
         return meta.existing.id;
+
     } else {
         filteredEntity.cid = shortid.generate();
-
+        meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
-        if (meta) {
-            meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
-        }
-
         return await _create(tx, listId, filteredEntity);
     }
 }
@@ -630,7 +671,7 @@ async function updateWithConsistencyCheck(context, listId, entity, source) {
             throw new interoperableErrors.ChangedError();
         }
 
-        await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, null, false);
+        await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, {}, false);
 
         const filteredEntity = filterObject(entity, allowedKeys);
 
@@ -638,7 +679,7 @@ async function updateWithConsistencyCheck(context, listId, entity, source) {
 
         updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
 
-        await _update(tx, listId, existing, filteredEntity);
+        await _update(tx, listId, groupedFieldsMap, existing, filteredEntity);
     });
 }
 
@@ -674,15 +715,13 @@ async function removeByEmailAndGet(context, listId, email) {
 async function _changeStatusTx(tx, context, listId, existing, newStatus) {
     enforce(newStatus !== SubscriptionStatus.SUBSCRIBED);
 
+    const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
+
     await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
 
-    await tx(getSubscriptionTableName(listId)).where('id', existing.id).update({
+    await _update(tx, listId, groupedFieldsMap, existing, {
         status: newStatus
     });
-
-    if (existing.status === SubscriptionStatus.SUBSCRIBED) {
-        await tx('lists').where('id', listId).decrement('subscribers', 1);
-    }
 }
 
 async function _unsubscribeExistingAndGetTx(tx, context, listId, existing) {
@@ -824,4 +863,5 @@ module.exports.unsubscribeByEmailAndGetTx = unsubscribeByEmailAndGetTx;
 module.exports.updateAddressAndGet = updateAddressAndGet;
 module.exports.updateManaged = updateManaged;
 module.exports.getListsWithEmail = getListsWithEmail;
-module.exports.changeStatusTx = changeStatusTx;
+module.exports.changeStatusTx = changeStatusTx;
+module.exports.purgeSensitiveData = purgeSensitiveData;

server/routes/rest/subscriptions.js

@@ -23,7 +23,7 @@ router.getAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, asy
 });
 
 router.postAsync('/subscriptions/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
-    return res.json(await subscriptions.create(req.context, castToInteger(req.params.listId), req.body, SubscriptionSource.ADMIN_FORM));
+    return res.json(await subscriptions.create(req.context, castToInteger(req.params.listId), req.body, SubscriptionSource.ADMIN_FORM, {}));
 });
 
 router.putAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, passport.csrfProtection, async (req, res) => {

server/routes/subscription.js

@@ -82,7 +82,7 @@ async function injectCustomFormData(customFormId, viewKey, data) {
         return;
     }
 
-    const form = await forms.getById(contextHelpers.getAdminContext(), customFormId);
+    const form = await forms.getById(contextHelpers.getAdminContext(), customFormId, false);
 
     data.template.template = form[viewKey] || data.template.template;
     data.template.layout = form.layout || data.template.layout;
@@ -539,7 +539,6 @@ router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req,
 
         data.isWeb = true;
         data.needsJsWarning = true;
-        data.isManagePreferences = true;
         data.flashMessages = await captureFlashMessages(res);
 
         res.send(htmlRenderer(data));
@@ -626,6 +625,11 @@ router.getAsync('/:cid/manual-unsubscribe-notice', async (req, res) => {
     await webNotice('manual-unsubscribe', req, res);
 });
 
+router.getAsync('/:cid/privacy-policy', async (req, res) => {
+    await webNotice('privacy-policy', req, res);
+});
+
+
 router.postAsync('/publickey', passport.parseForm, async (req, res) => {
     const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPassphrase', 'pgpPrivateKey']);
 
@@ -683,8 +687,6 @@ async function webNotice(type, req, res) {
     const htmlRenderer = await tools.getTemplate(data.template);
 
     data.isWeb = true;
-    data.isConfirmNotice = true; // FIXME: Not sure what this does. Check it in a browser with disabled JS
-    data.isManagePreferences = true;
     data.flashMessages = await captureFlashMessages(res);
 
     res.send(htmlRenderer(data));

server/services/gdpr-cleanup.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const config = require('config');
+const log = require('../lib/log');
+const knex = require('../lib/knex');
+const subscriptions = require('../models/subscriptions');
+const { SubscriptionStatus } = require('../../shared/lists');
+const contextHelpers = require('../lib/context-helpers');
+
+const checkPeriod = 60 * 1000;
+
+async function run() {
+    while (true) {
+        await knex.transaction(async tx => {
+            const currentTs = Date.now();
+
+            const lsts = await tx('lists').select(['id']);
+            for (const list of lsts) {
+
+                if (config.gdpr.deleteSubscriptionAfterUnsubscribe.enabled) {
+                    await tx(subscriptions.getSubscriptionTableName(list.id))
+                        .whereIn('status', [SubscriptionStatus.UNSUBSCRIBED, SubscriptionStatus.COMPLAINED])
+                        .where('unsubscribed', '<=', new Date(currentTs - config.gdpr.deleteSubscriptionAfterUnsubscribe.secondsAfterUnsubscribe * 1000))
+                        .del();
+                }
+
+                if (config.gdpr.deleteDataAfterUnsubscribe.enabled) {
+                    const groupedFieldsMap = await subscriptions.getGroupedFieldsMapTx(tx, list.id);
+
+                    const purgedEntity = {};
+                    subscriptions.purgeSensitiveData(purgedEntity, groupedFieldsMap);
+
+                    await tx(subscriptions.getSubscriptionTableName(list.id))
+                        .whereNotNull('email')
+                        .whereIn('status', [SubscriptionStatus.UNSUBSCRIBED, SubscriptionStatus.COMPLAINED])
+                        .where('unsubscribed', '<=', new Date(currentTs - config.gdpr.deleteDataAfterUnsubscribe.secondsAfterUnsubscribe * 1000))
+                        .update(purgedEntity);
+                }
+            }
+        });
+
+        const nextCycle = new Promise(resolve => {
+            setTimeout(resolve, checkPeriod);
+        });
+        await nextCycle;
+    }
+}
+
+function start() {
+    if (config.gdpr.deleteDataAfterUnsubscribe.enabled || config.gdpr.deleteDataAfterUnsubscribe.enabled) {
+        log.info('GDPR', 'Starting GDPR cleanup service');
+        run().catch(err => {
+            log.error('GDPR', err);
+        });
+    }
+}
+
+module.exports.start = start;

server/services/triggers.js

@@ -11,8 +11,8 @@ const { SubscriptionStatus } = require('../../shared/lists');
 const links = require('../models/links');
 const contextHelpers = require('../lib/context-helpers');
 
-const triggerCheckPeriod = 15 * 1000;
+const triggerCheckPeriod = 30 * 1000;
-const triggerFirePeriod = 60 * 1000;
+const triggerFirePeriod = 120 * 1000;
 
 
 async function run() {

server/setup/knex/migrations/20170506102634_v1_to_v2.js

@@ -241,6 +241,7 @@ async function migrateSubscriptions(knex) {
 
     const lists = await knex('lists');
     for (const list of lists) {
+        await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `unsubscribed` timestamp NULL DEFAULT NULL');
         await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `source_email` int(10) unsigned DEFAULT NULL');
         await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `hash_email` varchar(255) CHARACTER SET ascii');
 
@@ -286,6 +287,10 @@ async function migrateSubscriptions(knex) {
         }
 
         await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` MODIFY `hash_email` varchar(255) CHARACTER SET ascii NOT NULL');
+        await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` DROP KEY `email`');
+        await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` MODIFY `email` varchar(255) CHARACTER SET utf8 DEFAULT NULL');
+        await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD UNIQUE KEY `hash_email` (`hash_email`)');
+        await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD KEY `email` (`email`)');
 
         await knex.schema.table('subscription__' + list.id, table => {
             table.dropColumn('imported');
@@ -1128,8 +1133,10 @@ async function migrateTriggers(knex) {
         table.dropPrimary();
     });
 
+    // For some reason, .first() from Knex didn't work. So add the column directly via SQL to be able to specify that it should be the first in the table.
+    await knex.schema.raw('ALTER TABLE `queued` ADD `id` int(10) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
+
     await knex.schema.table('queued', table => {
-        table.increments('id').first().primary();
         table.renameColumn('subscriber', 'subscription');
         table.renameColumn('source', 'trigger');
     });

server/views/subscription/partials/subscription-flash-messages.hbs

@@ -1,11 +1,5 @@
 {{{flashMessages}}}
 
-{{#if isConfirmNotice}}
-    <div class="alert alert-warning js-warning" role="alert">
-        <strong>Warning!</strong> If JavaScript was not enabled then no confirmation message was sent
-    </div>
-{{/if}}
-
 {{#if needsJsWarning}}
     <div class="alert alert-danger js-warning" role="alert">
         <strong>Warning!</strong>

server/views/subscription/web-privacy-policy-notice.mjml.hbs

@@ -0,0 +1,13 @@
+<mj-section>
+    <mj-column>
+        <mj-text mj-class="h3">
+            Privacy Policy
+        </mj-text>
+        <mj-text mj-class="p">
+            How we use your data ...
+        </mj-text>
+        <mj-button mj-class="button" href="{{homepage}}">
+            Return to our website
+        </mj-button>
+    </mj-column>
+</mj-section>

shared/lists.js

@@ -28,7 +28,8 @@ const SubscriptionSource = {
     SUBSCRIPTION_FORM: -2,
     API: -3,
     NOT_IMPORTED_V1: -4,
-    IMPORTED_V1: -5
+    IMPORTED_V1: -5,
+    ERASED: -6
 };
 
 function getFieldColumn(field) {