public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implemented basic support for GDPR

Browse source
This commit is contained in:
Tomas Bures 2018-11-22 00:02:14 +03:00
parent 9f9cbc4c2b
commit 92ca1c0f28
21 changed files with 271 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

16
server/config/default.yaml
View file

@ -14,6 +14,20 @@
# Process title visible in monitoring logs and process listing
title: mailtrain
# GDPR related settings (https://en.wikipedia.org/wiki/General_Data_Protection_Regulation)
gdpr:
# If enabled, after a user unsubscribes or complains, his/her data are removed from the subscription.
# Only hashed email remains to prevent resubscription via import
deleteDataAfterUnsubscribe:
enabled: true
secondsAfterUnsubscribe: 86400 # 1 day
# If enabled, after the time below, an entry of a subscriber that unsubscribed or complained
# is completely removed from the subscribers list (including the hashed email)
deleteSubscriptionAfterUnsubscribe:
enabled: true
secondsAfterUnsubscribe: 2592000 # 30 days
# Enabled HTML editors
editors:
- grapesjs
@ -178,7 +192,7 @@ reports:
# performing network operations and in generating XSS attacks as part of the report.
# The bottom line is that if people who are creating report templates or have write access to the DB cannot be trusted,
# then it's safer to switch off the reporting functionality below.
enabled: false
enabled: true
testServer:
# Starts a vanity server that redirects all mail to /dev/null

2
server/index.js
View file

@ -6,6 +6,7 @@ const appBuilder = require('./app-builder');
const translate = require('./lib/translate');
const http = require('http');
const triggers = require('./services/triggers');
const gdprCleanup = require('./services/gdpr-cleanup');
const importer = require('./lib/importer');
const feedcheck = require('./lib/feedcheck');
const verpServer = require('./services/verp-server');
@ -105,6 +106,7 @@ dbcheck(err => { // Check if database needs upgrading before starting the server
feedcheck.spawn(() => {
senders.spawn(() => {
triggers.start();
gdprCleanup.start();
postfixBounceServer(async () => {
(async () => {

3
server/lib/client-helpers.js
View file

@ -41,7 +41,8 @@ async function getAuthenticatedConfig(context) {
globalPermissions,
editors: config.editors,
mosaico: config.mosaico,
verpEnabled: config.verp.enabled
verpEnabled: config.verp.enabled,
reportsEnabled: config.reports.enabled
}
}

6
server/models/blacklist.js
View file

@ -34,6 +34,8 @@ async function search(context, offset, limit, search) {
}
async function add(context, email) {
enforce(email, 'Email has to be set');
return await knex.transaction(async tx => {
shares.enforceGlobalPermission(context, 'manageBlacklist');
@ -45,11 +47,15 @@ async function add(context, email) {
}
async function remove(context, email) {
enforce(email, 'Email has to be set');
shares.enforceGlobalPermission(context, 'manageBlacklist');
await knex('blacklist').where('email', email).del();
}
async function isBlacklisted(email) {
enforce(email, 'Email has to be set');
const existing = await knex('blacklist').where('email', email).first();
return !!existing;
}

18
server/models/forms.js
View file

@ -9,7 +9,10 @@ const shares = require('./shares');
const namespaceHelpers = require('../lib/namespace-helpers');
const fs = require('fs-extra');
const path = require('path');
const mjml = require('mjml');
const mjml2html = mjml.default;
const lists = require('./lists');
const dependencyHelpers = require('../lib/dependency-helpers');
@ -43,7 +46,8 @@ const allowedFormKeys = new Set([
'web_unsubscribed_notice',
'mail_unsubscription_confirmed_html',
'mail_unsubscription_confirmed_text',
'web_manual_unsubscribe_notice'
'web_manual_unsubscribe_notice',
'web_privacy_policy_notice'
]);
const hashKeys = new Set([...formAllowedKeys, ...allowedFormKeys]);
@ -84,11 +88,15 @@ async function _getById(tx, id) {
}
async function getById(context, id) {
async function getById(context, id, withPermissions = true) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'customForm', id, 'view');
const entity = await _getById(tx, id);
entity.permissions = await shares.getPermissionsTx(tx, context, 'customForm', id);
if (withPermissions) {
entity.permissions = await shares.getPermissionsTx(tx, context, 'customForm', id);
}
return entity;
});
}
@ -219,7 +227,7 @@ function checkForMjmlErrors(form) {
let compiled;
try {
compiled = mjml(source);
compiled = mjml2html(source);
} catch (err) {
return err;
}
@ -230,7 +238,7 @@ function checkForMjmlErrors(form) {
const errors = {};
for (const key in form) {
if (key.startsWith('mail_') || key.startsWith('web_')) {
if ((key.startsWith('mail_') && key.endsWith('_html')) || key.startsWith('web_')) {
const template = form[key];
const errs = hasMjmlError(template);

5
server/models/lists.js
View file

@ -121,7 +121,7 @@ async function create(context, entity) {
await knex.schema.raw('CREATE TABLE `subscription__' + id + '` (\n' +
' `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
' `cid` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
' `email` varchar(255) CHARACTER SET utf8 NOT NULL,\n' +
' `email` varchar(255) CHARACTER SET utf8 DEFAULT NULL,\n' +
' `hash_email` varchar(255) CHARACTER SET ascii NOT NULL,\n' +
' `source_email` int(10) unsigned,\n' + // This references imports if the source is an import, 0 means some import in version 1, NULL if the source is via subscription or edit of the subscription
' `opt_in_ip` varchar(100) DEFAULT NULL,\n' +
@ -134,8 +134,9 @@ async function create(context, entity) {
' `latest_click` timestamp NULL DEFAULT NULL,\n' +
' `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n' +
' PRIMARY KEY (`id`),\n' +
' UNIQUE KEY `email` (`email`),\n' +
' UNIQUE KEY `hash_email` (`hash_email`),\n' +
' UNIQUE KEY `cid` (`cid`),\n' +
' KEY `email` (`email`),\n' +
' KEY `status` (`status`),\n' +
' KEY `subscriber_tz` (`tz`),\n' +
' KEY `is_test` (`is_test`),\n' +

106
server/models/subscriptions.js
View file

@ -1,5 +1,6 @@
'use strict';
const config = require('config');
const knex = require('../lib/knex');
const hasher = require('node-object-hash')();
const shortid = require('shortid');
@ -7,7 +8,7 @@ const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../../shared/interoperable-errors');
const shares = require('./shares');
const fields = require('./fields');
const { SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
const { SubscriptionSource, SubscriptionStatus, getFieldColumn } = require('../../shared/lists');
const segments = require('./segments');
const { enforce, filterObject } = require('../lib/helpers');
const moment = require('moment');
@ -475,7 +476,7 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
}
const existingWithKey = await existingWithKeyQuery.first();
if (existingWithKey) {
if (meta && (meta.updateAllowed || (meta.updateOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED))) {
if (existingWithKey.email === null || meta.updateAllowed || (meta.updateOfUnsubscribedAllowed && existingWithKey.status === SubscriptionStatus.UNSUBSCRIBED)) {
meta.update = true;
meta.existing = existingWithKey;
} else {
@ -486,12 +487,12 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
// The same for import where we need to subscribed only those (existing and new) that have not been unsubscribed already.
// In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status to SUBSCRIBED
if (meta && meta.subscribeIfNoExisting && !entity.status) {
if (meta.subscribeIfNoExisting && !entity.status) {
entity.status = SubscriptionStatus.SUBSCRIBED;
}
}
if ((isCreate && !(meta && meta.update)) || 'status' in entity) {
if ((isCreate && !meta.update) || 'status' in entity) {
enforce(entity.status >= SubscriptionStatus.MIN && entity.status <= SubscriptionStatus.MAX, 'Invalid status');
}
@ -532,26 +533,69 @@ function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
}
}
async function _update(tx, listId, existing, filteredEntity) {
function purgeSensitiveData(subscription, groupedFieldsMap) {
subscription.email = null;
for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) {
for (const optionKey in fld.groupedOptions) {
const option = fld.groupedOptions[optionKey];
subscription[option.column] = null;
subscription['source_' + option.column] = SubscriptionSource.ERASED;
}
} else {
subscription[fldCol] = null;
subscription['source_' + fldCol] = SubscriptionSource.ERASED;
}
}
}
async function _update(tx, listId, groupedFieldsMap, existing, filteredEntity) {
if ('status' in filteredEntity) {
if (existing.status !== filteredEntity.status) {
filteredEntity.status_change = new Date();
}
}
console.log(filteredEntity);
await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
if ('status' in filteredEntity) {
let countIncrement = 0;
if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
countIncrement = -1;
} else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
countIncrement = 1;
if (filteredEntity.status === SubscriptionStatus.UNSUBSCRIBED || filteredEntity.status === SubscriptionStatus.COMPLAINED) {
if (existing.unsubscribed === null) {
filteredEntity.unsubscribed = new Date();
}
if (countIncrement) {
await tx('lists').where('id', listId).increment('subscribers', countIncrement);
if (config.gdpr.deleteSubscriptionAfterUnsubscribe.enabled && config.gdpr.deleteSubscriptionAfterUnsubscribe.secondsAfterUnsubscribe === 0) {
filteredEntity = null;
} else if (config.gdpr.deleteDataAfterUnsubscribe.enabled && config.gdpr.deleteDataAfterUnsubscribe.secondsAfterUnsubscribe === 0) {
purgeSensitiveData(filteredEntity, groupedFieldsMap);
}
} else if (filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
filteredEntity.unsubscribed = null;
}
if (filteredEntity) {
await tx(getSubscriptionTableName(listId)).where('id', existing.id).update(filteredEntity);
if ('status' in filteredEntity) {
let countIncrement = 0;
if (existing.status === SubscriptionStatus.SUBSCRIBED && filteredEntity.status !== SubscriptionStatus.SUBSCRIBED) {
countIncrement = -1;
} else if (existing.status !== SubscriptionStatus.SUBSCRIBED && filteredEntity.status === SubscriptionStatus.SUBSCRIBED) {
countIncrement = 1;
}
if (countIncrement) {
await tx('lists').where('id', listId).increment('subscribers', countIncrement);
}
}
} else {
await tx(getSubscriptionTableName(listId)).where('id', existing.id).del();
if (existing.status === SubscriptionStatus.SUBSCRIBED) {
await tx('lists').where('id', listId).decrement('subscribers', 1);
}
}
}
@ -586,20 +630,17 @@ async function createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMa
updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
filteredEntity.opt_in_ip = meta && meta.ip;
filteredEntity.opt_in_country = meta && meta.country;
filteredEntity.opt_in_ip = meta.ip;
filteredEntity.opt_in_country = meta.country;
if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
await _update(tx, listId, meta.existing, filteredEntity);
if (meta.update) { // meta.update is set by _validateAndPreprocess
await _update(tx, listId, groupedFieldsMap, meta.existing, filteredEntity);
meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
return meta.existing.id;
} else {
filteredEntity.cid = shortid.generate();
if (meta) {
meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
}
meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
return await _create(tx, listId, filteredEntity);
}
}
@ -630,7 +671,7 @@ async function updateWithConsistencyCheck(context, listId, entity, source) {
throw new interoperableErrors.ChangedError();
}
await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, null, false);
await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, {}, false);
const filteredEntity = filterObject(entity, allowedKeys);
@ -638,7 +679,7 @@ async function updateWithConsistencyCheck(context, listId, entity, source) {
updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
await _update(tx, listId, existing, filteredEntity);
await _update(tx, listId, groupedFieldsMap, existing, filteredEntity);
});
}
@ -674,15 +715,13 @@ async function removeByEmailAndGet(context, listId, email) {
async function _changeStatusTx(tx, context, listId, existing, newStatus) {
enforce(newStatus !== SubscriptionStatus.SUBSCRIBED);
const groupedFieldsMap = await getGroupedFieldsMapTx(tx, listId);
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
await tx(getSubscriptionTableName(listId)).where('id', existing.id).update({
await _update(tx, listId, groupedFieldsMap, existing, {
status: newStatus
});
if (existing.status === SubscriptionStatus.SUBSCRIBED) {
await tx('lists').where('id', listId).decrement('subscribers', 1);
}
}
async function _unsubscribeExistingAndGetTx(tx, context, listId, existing) {
@ -824,4 +863,5 @@ module.exports.unsubscribeByEmailAndGetTx = unsubscribeByEmailAndGetTx;
module.exports.updateAddressAndGet = updateAddressAndGet;
module.exports.updateManaged = updateManaged;
module.exports.getListsWithEmail = getListsWithEmail;
module.exports.changeStatusTx = changeStatusTx;
module.exports.changeStatusTx = changeStatusTx;
module.exports.purgeSensitiveData = purgeSensitiveData;

2
server/routes/rest/subscriptions.js
View file

@ -23,7 +23,7 @@ router.getAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, asy
});
router.postAsync('/subscriptions/:listId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
return res.json(await subscriptions.create(req.context, castToInteger(req.params.listId), req.body, SubscriptionSource.ADMIN_FORM));
return res.json(await subscriptions.create(req.context, castToInteger(req.params.listId), req.body, SubscriptionSource.ADMIN_FORM, {}));
});
router.putAsync('/subscriptions/:listId/:subscriptionId', passport.loggedIn, passport.csrfProtection, async (req, res) => {

10
server/routes/subscription.js
View file

@ -82,7 +82,7 @@ async function injectCustomFormData(customFormId, viewKey, data) {
return;
}
const form = await forms.getById(contextHelpers.getAdminContext(), customFormId);
const form = await forms.getById(contextHelpers.getAdminContext(), customFormId, false);
data.template.template = form[viewKey] || data.template.template;
data.template.layout = form.layout || data.template.layout;
@ -539,7 +539,6 @@ router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req,
data.isWeb = true;
data.needsJsWarning = true;
data.isManagePreferences = true;
data.flashMessages = await captureFlashMessages(res);
res.send(htmlRenderer(data));
@ -626,6 +625,11 @@ router.getAsync('/:cid/manual-unsubscribe-notice', async (req, res) => {
await webNotice('manual-unsubscribe', req, res);
});
router.getAsync('/:cid/privacy-policy', async (req, res) => {
await webNotice('privacy-policy', req, res);
});
router.postAsync('/publickey', passport.parseForm, async (req, res) => {
const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPassphrase', 'pgpPrivateKey']);
@ -683,8 +687,6 @@ async function webNotice(type, req, res) {
const htmlRenderer = await tools.getTemplate(data.template);
data.isWeb = true;
data.isConfirmNotice = true; // FIXME: Not sure what this does. Check it in a browser with disabled JS
data.isManagePreferences = true;
data.flashMessages = await captureFlashMessages(res);
res.send(htmlRenderer(data));

58
server/services/gdpr-cleanup.js Normal file
View file

@ -0,0 +1,58 @@
'use strict';
const config = require('config');
const log = require('../lib/log');
const knex = require('../lib/knex');
const subscriptions = require('../models/subscriptions');
const { SubscriptionStatus } = require('../../shared/lists');
const contextHelpers = require('../lib/context-helpers');
const checkPeriod = 60 * 1000;
async function run() {
while (true) {
await knex.transaction(async tx => {
const currentTs = Date.now();
const lsts = await tx('lists').select(['id']);
for (const list of lsts) {
if (config.gdpr.deleteSubscriptionAfterUnsubscribe.enabled) {
await tx(subscriptions.getSubscriptionTableName(list.id))
.whereIn('status', [SubscriptionStatus.UNSUBSCRIBED, SubscriptionStatus.COMPLAINED])
.where('unsubscribed', '<=', new Date(currentTs - config.gdpr.deleteSubscriptionAfterUnsubscribe.secondsAfterUnsubscribe * 1000))
.del();
}
if (config.gdpr.deleteDataAfterUnsubscribe.enabled) {
const groupedFieldsMap = await subscriptions.getGroupedFieldsMapTx(tx, list.id);
const purgedEntity = {};
subscriptions.purgeSensitiveData(purgedEntity, groupedFieldsMap);
await tx(subscriptions.getSubscriptionTableName(list.id))
.whereNotNull('email')
.whereIn('status', [SubscriptionStatus.UNSUBSCRIBED, SubscriptionStatus.COMPLAINED])
.where('unsubscribed', '<=', new Date(currentTs - config.gdpr.deleteDataAfterUnsubscribe.secondsAfterUnsubscribe * 1000))
.update(purgedEntity);
}
}
});
const nextCycle = new Promise(resolve => {
setTimeout(resolve, checkPeriod);
});
await nextCycle;
}
}
function start() {
if (config.gdpr.deleteDataAfterUnsubscribe.enabled || config.gdpr.deleteDataAfterUnsubscribe.enabled) {
log.info('GDPR', 'Starting GDPR cleanup service');
run().catch(err => {
log.error('GDPR', err);
});
}
}
module.exports.start = start;

4
server/services/triggers.js
View file

@ -11,8 +11,8 @@ const { SubscriptionStatus } = require('../../shared/lists');
const links = require('../models/links');
const contextHelpers = require('../lib/context-helpers');
const triggerCheckPeriod = 15 * 1000;
const triggerFirePeriod = 60 * 1000;
const triggerCheckPeriod = 30 * 1000;
const triggerFirePeriod = 120 * 1000;
async function run() {

9
server/setup/knex/migrations/20170506102634_v1_to_v2.js
View file

@ -241,6 +241,7 @@ async function migrateSubscriptions(knex) {
const lists = await knex('lists');
for (const list of lists) {
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `unsubscribed` timestamp NULL DEFAULT NULL');
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `source_email` int(10) unsigned DEFAULT NULL');
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `hash_email` varchar(255) CHARACTER SET ascii');
@ -286,6 +287,10 @@ async function migrateSubscriptions(knex) {
}
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` MODIFY `hash_email` varchar(255) CHARACTER SET ascii NOT NULL');
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` DROP KEY `email`');
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` MODIFY `email` varchar(255) CHARACTER SET utf8 DEFAULT NULL');
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD UNIQUE KEY `hash_email` (`hash_email`)');
await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD KEY `email` (`email`)');
await knex.schema.table('subscription__' + list.id, table => {
table.dropColumn('imported');
@ -1128,8 +1133,10 @@ async function migrateTriggers(knex) {
table.dropPrimary();
});
// For some reason, .first() from Knex didn't work. So add the column directly via SQL to be able to specify that it should be the first in the table.
await knex.schema.raw('ALTER TABLE `queued` ADD `id` int(10) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await knex.schema.table('queued', table => {
table.increments('id').first().primary();
table.renameColumn('subscriber', 'subscription');
table.renameColumn('source', 'trigger');
});

6
server/views/subscription/partials/subscription-flash-messages.hbs
View file

@ -1,11 +1,5 @@
{{{flashMessages}}}
{{#if isConfirmNotice}}
<div class="alert alert-warning js-warning" role="alert">
<strong>Warning!</strong> If JavaScript was not enabled then no confirmation message was sent
</div>
{{/if}}
{{#if needsJsWarning}}
<div class="alert alert-danger js-warning" role="alert">
<strong>Warning!</strong>

13
server/views/subscription/web-privacy-policy-notice.mjml.hbs Normal file
View file

@ -0,0 +1,13 @@
<mj-section>
<mj-column>
<mj-text mj-class="h3">
Privacy Policy
</mj-text>
<mj-text mj-class="p">
How we use your data ...
</mj-text>
<mj-button mj-class="button" href="{{homepage}}">
Return to our website
</mj-button>
</mj-column>
</mj-section>