public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

WiP on mailers

Browse source
This commit is contained in:
Tomas Bures 2018-04-29 18:13:40 +02:00
parent e97415c237
commit a4ee1534cc
46 changed files with 1263 additions and 529 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
app-builder.js
View file

@ -20,7 +20,6 @@ const hbs = require('hbs');
const handlebarsHelpers = require('./lib/handlebars-helpers');
const compression = require('compression');
const passport = require('./lib/passport');
const tools = require('./lib/tools');
const contextHelpers = require('./lib/context-helpers');
const getSettings = nodeifyFunction(require('./models/settings').get);

216
client/public/subscription/form-input-style.css Normal file
View file

@ -0,0 +1,216 @@
/* --- Colors ----------
Input Border: #DCE4EC
Input Group: #FAFAFA
Muted: #999999
Anchor: #1F68D5
Alerts: ...
*/
/* --- General -------- */
form {
margin: .5em 0 1em;
}
.form-group {
margin-bottom: 1.2em;
box-sizing: border-box;
}
input,
select,
textarea {
margin-bottom: 0;
}
label {
display: block;
font-size: 1em;
font-weight: 700;
margin-bottom: .3em;
}
.label-checkbox,
.label-radio {
font-weight: normal;
font-size: .9em;
}
.label-inline {
display: inline-block;
font-weight: normal;
margin-left: .3em;
}
/* --- Inputs ------------- */
input[type='email'],
input[type='number'],
input[type='password'],
input[type='search'],
input[type='tel'],
input[type='text'],
input[type='url'],
textarea,
select {
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
background-color: transparent;
border: 2px solid #DCE4EC;
border-radius: 4px;
box-shadow: none;
box-sizing: border-box;
height: 2.8em;
padding: .3em .7em;
width: 100%;
font-size: 1.2em;
font-style: inherit;
}
input[type='email']:focus,
input[type='number']:focus,
input[type='password']:focus,
input[type='search']:focus,
input[type='tel']:focus,
input[type='text']:focus,
input[type='url']:focus,
textarea:focus,
select:focus {
border-color: #2D3E4F;
outline: 0;
}
input[readonly] {
color: #999999;
}
input[readonly]:focus {
border-color: #DCE4EC;
}
input[type="checkbox"],
input[type="radio"] {
margin-bottom: 0;
margin-right: .2em;
}
input[type='checkbox'],
input[type='radio'] {
display: inline;
}
select {
background: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" height="14" viewBox="0 0 29 14" width="29"><path fill="#d1d1d1" d="M9.37727 3.625l5.08154 6.93523L19.54036 3.625"/></svg>') center right no-repeat;
padding-right: 2em;
}
select:focus {
background-image: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" height="14" viewBox="0 0 29 14" width="29"><path fill="#2D3E4F" d="M9.37727 3.625l5.08154 6.93523L19.54036 3.625"/></svg>');
}
textarea {
min-height: 8em;
max-width: 100%;
}
/* --- Input Group --------- */
.input-group {
position: relative;
}
.input-group-addon {
position: absolute;
right: 0;
top: 0;
background: #FAFAFA;
height: 100%;
padding: 0 .75em;
border: 2px solid #DCE4EC;
box-sizing: border-box;
border-bottom-right-radius: 4px;
border-top-right-radius: 4px;
}
.input-group-addon > * {
line-height: 2.8em;
}
/* --- Alerts ------------- */
.alert {
margin: 22px auto 0;
padding: 15px;
border-radius: 4px;
font-family: inherit;
font-size: 15px;
line-height: 21px;
box-sizing: border-box;
}
.alert-dismissible .close {
display: none;
}
.alert-success { color: #397740; background-color: #DEF0D9; border-color: #CFEAC8; }
.alert-info { color: #33708E; background-color: #D9EDF6; border-color: #BCDFF0; }
.alert-warning { color: #8A6D3F; background-color: #FCF8E4; border-color: #F9F2CE; }
.alert-danger { color: #AA4144; background-color: #F2DEDE; border-color: #EBCCCC; }
/* --- GPG Key ------------- */
.form-group.gpg > label {
display: inline-block;
}
.btn-download-pubkey,
.btn-download-pubkey:focus,
.btn-download-pubkey:active {
background: none;
border: none;
display: block;
font: inherit;
font-size: .8em;
margin: .3em 0 0;
padding: 0;
outline: none;
outline-offset: 0;
color: #1F68D5;
cursor: pointer;
text-transform: none;
height: auto;
float: right;
}
.btn-download-pubkey:hover {
text-decoration: underline;
}
.gpg-text {
font-family: monospace;
font-size: .8em;
}
/* --- Other ------------- */
.help-block {
display: block;
font-size: .9em;
line-height: 1;
color: #999999;
}
form a {
color: #1F68D5;
text-decoration: none;
}
form a:hover {
text-decoration: underline;
}

176
client/public/subscription/widget.js Normal file
View file

@ -0,0 +1,176 @@
/* eslint-env browser */
/* eslint prefer-arrow-callback: 0, object-shorthand: 0, new-cap: 0, no-invalid-this: 0, no-var: 0*/
if (typeof window.mailtrain !== 'object') {
window.mailtrain = {};
(function(mt) {
'use strict';
if (document.documentMode <= 9 || typeof XMLHttpRequest === 'undefined') {
return;
}
var cookie = {
create: function(name, value, days) {
var date = new Date();
date.setTime(date.getTime() + ((days || 365) * 24 * 60 * 60 * 1000));
var expires = '; expires=' + date.toGMTString();
document.cookie = name + '=' + value + expires + '; path=/';
},
read: function(name) {
var a = document.cookie.match('(^|;)\\s*' + name + '\\s*=\\s*([^;]+)');
return a ? a.pop() : null;
}
};
var forEach = function(array, callback, scope) {
for (var i = 0; i < array.length; i++) {
callback.call(scope, i, array[i]);
}
};
var loadScript = function(exists, url, callback) {
if (eval(exists)) {
return callback(true);
}
var scriptTag = document.createElement('script');
scriptTag.setAttribute('type', 'text/javascript');
scriptTag.setAttribute('src', url);
if (typeof callback !== 'undefined') {
if (scriptTag.readyState) {
scriptTag.onreadystatechange = function() {
(this.readyState === 'complete' || this.readyState === 'loaded') && callback();
};
} else {
scriptTag.onload = callback;
}
}
(document.getElementsByTagName('head')[0] || document.documentElement).appendChild(scriptTag);
};
var serialize = function(form) {
var field, s = [];
if (typeof form === 'object' && form.nodeName === 'FORM') {
var len = form.elements.length;
for (var i = 0; i < len; i++) {
field = form.elements[i];
if (field.name && !field.disabled && field.type !== 'file' && field.type !== 'reset' && field.type !== 'submit' && field.type !== 'button') {
if (field.type === 'select-multiple') {
for (var j = form.elements[i].options.length - 1; j >= 0; j--) {
if (field.options[j].selected) {
s[s.length] = encodeURIComponent(field.name) + '=' + encodeURIComponent(field.options[j].value);
}
}
} else if ((field.type !== 'checkbox' && field.type !== 'radio') || field.checked) {
s[s.length] = encodeURIComponent(field.name) + '=' + encodeURIComponent(field.value);
}
}
}
}
return s.join('&').replace(/%20/g, '+');
};
var getXHR = function(method, url, successHandler, errorHandler) {
var xhr = new XMLHttpRequest();
xhr.open(method, url, true);
xhr.onreadystatechange = function() {
if (xhr.readyState === 4) {
var data;
try {
data = JSON.parse(xhr.responseText);
} catch(err) {
data = { error: err.message || err };
}
xhr.status === 200 && !data.error
? successHandler && successHandler(data)
: errorHandler && errorHandler(data);
}
};
return xhr;
};
var getJSON = function(url, successHandler, errorHandler) {
var xhr = getXHR('get', url, successHandler, errorHandler);
xhr.send();
};
var sendForm = function(form, successHandler, errorHandler) {
var url = form.getAttribute('action');
var xhr = getXHR('post', url, successHandler, errorHandler);
xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
xhr.setRequestHeader('Content-type', 'application/x-www-form-urlencoded');
xhr.send(serialize(form));
};
var renderWidget = function(container, data) {
container.innerHTML = data.html;
var form = container.querySelector('.form');
var statusContainer = container.querySelector('.status');
var setStatus = function(templateName, message) {
var html = container.querySelector('div[data-status-template="' + templateName + '"]').outerHTML;
html = message ? html.replace('{message}', message) : html;
statusContainer.innerHTML = html;
};
if (cookie.read('has-subscribed-to-' + data.cid)) {
setStatus('already-subscribed');
form.style.display = 'none';
return;
}
container.querySelector('.sub-time').value = new Date().getTime();
if (window.moment && window.moment.tz) {
container.querySelector('.tz-detect').value = window.moment.tz.guess() || '';
}
var isSending = false;
form.addEventListener('submit', function(event) {
event.preventDefault();
if (isSending) {
return;
}
isSending = true;
setStatus('spinner');
sendForm(form, function(j) {
isSending = false;
setStatus('confirm-notice');
form.style.display = 'none';
container.scrollIntoView();
cookie.create('has-subscribed-to-' + data.cid, 1);
}, function(j) {
isSending = false;
setStatus('error', j.error);
});
});
};
forEach(document.body.querySelectorAll('div[data-mailtrain-subscription-widget]'), function(i, container) {
var url = container.getAttribute('data-url');
getJSON(url, function(j) {
renderWidget(container, j.data);
}, function(j) {
console.log(j);
});
});
document.addEventListener('DOMContentLoaded', function() {
loadScript('window.moment', 'https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.12.0/moment.min.js', function(existed) {
loadScript('window.moment.tz', 'https://cdnjs.cloudflare.com/ajax/libs/moment-timezone/0.5.3/moment-timezone-with-data.min.js', function(existed) {
if (window.moment && window.moment.tz) {
forEach(document.body.querySelectorAll('div[data-mailtrain-subscription-widget] .tz-detect'), function(i, el) {
el.value = window.moment.tz.guess() || '';
});
}
});
});
});
})(window.mailtrain);
}

13
client/src/lists/CUD.js
View file

@ -14,6 +14,7 @@ import { validateNamespace, NamespaceSelect } from '../lib/namespace';
import { UnsubscriptionMode } from '../../../shared/lists';
import styles from "../lib/styles.scss";
import mailtrainConfig from 'mailtrainConfig';
import {getMailerTypes} from "../send-configurations/helpers";
@translate()
@withForm
@ -27,6 +28,8 @@ export default class CUD extends Component {
this.state = {};
this.initForm();
this.mailerTypes = getMailerTypes(props.t);
}
static propTypes = {
@ -148,6 +151,13 @@ export default class CUD extends Component {
{data: 3, title: t('Namespace')}
];
const sendConfigurationsColumns = [
{ data: 1, title: t('Name') },
{ data: 2, title: t('Description') },
{ data: 3, title: t('Type'), render: data => this.mailerTypes[data].typeName },
{ data: 5, title: t('Namespace') }
];
return (
<div>
{canDelete &&
@ -176,13 +186,14 @@ export default class CUD extends Component {
<InputField id="contact_email" label={t('Contact email')} help={t('Contact email used in subscription forms and emails that are sent out. If not filled in, the admin email from the global settings will be used.')}/>
<InputField id="homepage" label={t('Homepage')} help={t('Homepage URL used in subscription forms and emails that are sent out. If not filled in, the default homepage from global settings will be used.')}/>
<TableSelect id="send_configuration" label={t('Send configuration')} withHeader dropdown dataUrl='/rest/send-configurations-table' columns={sendConfigurationsColumns} selectionLabelIndex={1} help={t('Send configuration that will be used for sending out subscription-related emails.')}/>
<NamespaceSelect/>
<Dropdown id="form" label={t('Forms')} options={formsOptions} help={t('Web and email forms and templates used in subscription management process.')}/>
{this.getFormValue('form') === 'custom' &&
<TableSelect id="default_form" label={t('Custom Forms')} withHeader dropdown dataUrl='/rest/forms-table' columns={customFormsColumns} selectionLabelIndex={1} help={<Trans>The custom form used for this list. You can create a form <a href={`/lists/forms/create/${this.props.entity.id}`}>here</a>.</Trans>}/>
<TableSelect id="default_form" label={t('Custom forms')} withHeader dropdown dataUrl='/rest/forms-table' columns={customFormsColumns} selectionLabelIndex={1} help={<Trans>The custom form used for this list. You can create a form <a href={`/lists/forms/create/${this.props.entity.id}`}>here</a>.</Trans>}/>
}
<CheckBox id="public_subscribe" label={t('Subscription')} text={t('Allow public users to subscribe themselves')}/>

41
index.js
View file

@ -4,15 +4,15 @@ const config = require('config');
const log = require('npmlog');
const appBuilder = require('./app-builder');
const http = require('http');
const triggers = require('./services/triggers');
const importer = require('./services/importer');
const verpServer = require('./services/verp-server');
//const triggers = require('./services/triggers');
// const importer = require('./services/importer');
// const verpServer = require('./services/verp-server');
const testServer = require('./services/test-server');
const postfixBounceServer = require('./services/postfix-bounce-server');
//const postfixBounceServer = require('./services/postfix-bounce-server');
const tzupdate = require('./services/tzupdate');
const feedcheck = require('./services/feedcheck');
//const feedcheck = require('./services/feedcheck');
const dbcheck = require('./lib/dbcheck');
const senders = require('./lib/senders');
//const senders = require('./lib/senders');
const reportProcessor = require('./lib/report-processor');
const executor = require('./lib/executor');
const privilegeHelpers = require('./lib/privilege-helpers');
@ -83,28 +83,29 @@ dbcheck(err => { // Check if database needs upgrading before starting the server
.then(() =>
executor.spawn(() => {
testServer(() => {
verpServer(() => {
//verpServer(() => {
startHTTPServer(true, trustedPort, () => {
startHTTPServer(false, sandboxPort, () => {
privilegeHelpers.dropRootPrivileges();
tzupdate(() => {
importer(() => {
triggers(() => {
senders.spawn(() => {
feedcheck(() => {
postfixBounceServer(async () => {
tzupdate.start();
//importer(() => {
//triggers(() => {
//senders.spawn(() => {
//feedcheck(() => {
//postfixBounceServer(async () => {
(async () => {
await reportProcessor.init();
log.info('Service', 'All services started');
})();
//});
//});
//});
//});
//});
});
});
});
});
});
});
});
});
});
//});
});
})
);

232
lib/mailers.js Normal file
View file

@ -0,0 +1,232 @@
'use strict';
const log = require('npmlog');
const config = require('config');
const Handlebars = require('handlebars');
const util = require('util');
const nodemailer = require('nodemailer');
const aws = require('aws-sdk');
const openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt;
const sendConfigurations = require('../models/send-configurations');
const contextHelpers = require('./context-helpers');
const settings = require('../models/settings');
const tools = require('./tools');
const htmlToText = require('html-to-text');
const bluebird = require('bluebird');
const _ = require('./translate')._;
Handlebars.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
if (typeof options === 'undefined' && context) {
options = context;
context = false;
}
let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
if (Array.isArray(context)) {
result = util.format(result, ...context);
}
return new Handlebars.SafeString(result);
});
const transports = new Map();
async function getOrCreateMailer(sendConfiguration) {
if (!sendConfiguration) {
sendConfiguration = sendConfigurations.getSystemSendConfiguration();
}
const transport = transports.get(sendConfiguration.id) || await _createTransport(sendConfiguration);
return transport.mailer;
}
function invalidateMailer(sendConfiguration) {
transports.delete(sendConfiguration.id);
}
async function _sendMail(transport, mail, template) {
if (!mail.headers) {
mail.headers = {};
}
mail.headers['X-Sending-Zone'] = 'transactional';
const htmlRenderer = await tools.getTemplate(template.html);
if (htmlRenderer) {
mail.html = htmlRenderer(template.data || {});
}
const preparedHtml = await tools.prepareHtml(mail.html);
if (prepareHtml) {
mail.html = prepareHtml;
}
const textRenderer = await tools.getTemplate(template.text);
if (textRenderer) {
mail.text = textRenderer(template.data || {});
} else if (mail.html) {
mail.text = htmlToText.fromString(mail.html, {
wordwrap: 130
});
}
let tryCount = 0;
const trySend = (callback) => {
tryCount++;
transport.sendMail(mail, (err, info) => {
if (err) {
log.error('Mail', err);
if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) {
// temporary error, try again
log.verbose('Mail', 'Retrying after %s sec. ...', tryCount);
return setTimeout(trySend, tryCount * 1000);
}
return callback(err);
}
return callback(null, info);
});
};
const trySendAsync = bluebird.promisify(trySend);
return await trySendAsync();
}
async function _createTransport(sendConfiguration) {
const mailerSettings = sendConfiguration.mailer_settings;
const mailerType = sendConfiguration.mailer_type;
const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey', 'pgpPassphrase']);
const existingTransport = transports.get(sendConfiguration.id);
let existingListeners = [];
if (existingTransport) {
existingListeners = existingTransport.listeners('idle');
existingTransport.removeAllListeners('idle');
existingTransport.removeAllListeners('stream');
existingTransport.checkThrottling = null;
}
const logFunc = (...args) => {
const level = args.shift();
args.shift();
args.unshift('Mail');
log[level](...args);
};
let transportOptions;
if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
transportOptions = {
pool: true,
host: mailerSettings.hostname,
port: mailerSettings.port || false,
secure: mailerSettings.encryption === 'TLS',
ignoreTLS: mailerSettings.encryption === 'NONE',
auth: mailerSettings.useAuth ? {
user: mailerSettings.user,
pass: mailerSettings.password
} : false,
debug: mailerSettings.logTransactions,
logger: mailerSettings.logTransactions ? {
debug: logFunc.bind(null, 'verbose'),
info: logFunc.bind(null, 'info'),
error: logFunc.bind(null, 'error')
} : false,
maxConnections: mailerSettings.maxConnections,
maxMessages: mailerSettings.maxMessages,
tls: {
rejectUnauthorized: !mailerSettings.allowSelfSigned
}
};
} else if (mailerType === sendConfigurations.MailerType.AWS_SES) {
const sendingRate = mailerSettings.throttling / 3600; // convert to messages/second
transportOptions = {
SES: new aws.SES({
apiVersion: '2010-12-01',
accessKeyId: mailerSettings.key,
secretAccessKey: mailerSettings.secret,
region: mailerSettings.region
}),
debug: mailerSettings.logTransactions,
logger: mailerSettings.logTransactions ? {
debug: logFunc.bind(null, 'verbose'),
info: logFunc.bind(null, 'info'),
error: logFunc.bind(null, 'error')
} : false,
maxConnections: mailerSettings.maxConnections,
sendingRate
};
} else {
throw new Error('Invalid mail transport');
}
const transport = nodemailer.createTransport(transportOptions, config.nodemailer);
transport.use('stream', openpgpEncrypt({
signingKey: configItems.pgpPrivateKey,
passphrase: configItems.pgpPassphrase
}));
if (existingListeners.length) {
log.info('Mail', 'Reattaching %s idle listeners', existingListeners.length);
existingListeners.forEach(listener => transport.on('idle', listener));
}
let checkThrottling;
if (mailerType === sendConfigurations.MailerType.GENERIC_SMTP || mailerType === sendConfigurations.MailerType.ZONE_MTA) {
let throttling = mailerSettings.throttling;
if (throttling) {
throttling = 1 / (throttling / (3600 * 1000));
}
let lastCheck = Date.now();
checkThrottling = function (next) {
if (!throttling) {
return next();
}
let nextCheck = Date.now();
let checkDiff = (nextCheck - lastCheck);
if (checkDiff < throttling) {
log.verbose('Mail', 'Throttling next message in %s sec.', (throttling - checkDiff) / 1000);
setTimeout(() => {
lastCheck = Date.now();
next();
}, throttling - checkDiff);
} else {
lastCheck = nextCheck;
next();
}
};
} else {
checkThrottling = next => next();
}
transport.mailer = {
checkThrottling,
sendMail: async (mail, template) => await _sendMail(transport, mail, template)
};
transports.set(sendConfiguration.id, transport);
return transport;
}
module.exports = {
getOrCreateMailer,
invalidateMailer
};

2
lib/report-processor.js
View file

@ -8,7 +8,7 @@ const contextHelpers = require('../lib/context-helpers');
let runningWorkersCount = 0;
let maxWorkersCount = 1;
let workers = {};
const workers = {};
function startWorker(report) {

24
lib/subscription-mail-helpers.js
View file

@ -3,16 +3,13 @@
const log = require('npmlog');
const fields = require('../models/fields');
const settings = require('../models/settings');
const urllib = require('url');
const helpers = require('./helpers');
const {getTrustedUrl} = require('./urls');
const _ = require('./translate')._;
const util = require('util');
const contextHelpers = require('./context-helpers');
const {getFieldKey} = require('../shared/lists');
const forms = require('../models/forms');
const bluebird = require('bluebird');
const sendMail = bluebird.promisify(require('./mailer').sendMail);
const mailers = require('./mailers');
module.exports = {
sendAlreadySubscribed,
@ -99,8 +96,6 @@ function getDisplayName(flds, subscription) {
}
async function _sendMail(list, email, template, subject, relativeUrls, subscription) {
console.log(subscription);
const flds = await fields.list(contextHelpers.getAdminContext(), list.id);
const encryptionKeys = [];
@ -110,16 +105,16 @@ async function _sendMail(list, email, template, subject, relativeUrls, subscript
}
}
const configItems = await settings.get(['defaultHomepage', 'defaultFrom', 'defaultAddress', 'serviceUrl']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['defaultHomepage', 'adminEmail']);
const data = {
title: list.name,
homepage: configItems.defaultHomepage || configItems.serviceUrl,
contactAddress: configItems.defaultAddress,
homepage: configItems.defaultHomepage || getTrustedUrl(),
contactAddress: list.from_email || configItems.adminEmail,
};
for (let relativeUrlKey in relativeUrls) {
data[relativeUrlKey] = urllib.resolve(configItems.serviceUrl, relativeUrls[relativeUrlKey]);
data[relativeUrlKey] = getTrustedUrl(relativeUrls[relativeUrlKey]);
}
const fsTemplate = template.replace(/_/g, '-');
@ -142,7 +137,9 @@ async function _sendMail(list, email, template, subject, relativeUrls, subscript
}
try {
await sendMail({
if (list.send_configuration) {
const mailer = await mailers.getOrCreateMailer(list.send_configuration);
await mailer.sendMail({
from: {
name: configItems.defaultFrom,
address: configItems.defaultAddress
@ -158,6 +155,9 @@ async function _sendMail(list, email, template, subject, relativeUrls, subscript
text,
data
});
} else {
log.warn('Subscription', `Not sending email for list id:${list.id} because not send configuration is set.`);
}
} catch (err) {
log.error('Subscription', err);
}

50
lib/tools-async.js
View file

@ -1,50 +0,0 @@
'use strict';
const _ = require('./translate')._;
const util = require('util');
const isemail = require('isemail');
const bluebird = require('bluebird');
const mergeTemplateIntoLayout = bluebird.promisify(require('./tools').mergeTemplateIntoLayout);
module.exports = {
validateEmail,
validateEmailGetMessage,
mergeTemplateIntoLayout
};
async function validateEmail(address, checkBlocked) {
let user = (address || '').toString().split('@').shift().toLowerCase().replace(/[^a-z0-9]/g, '');
if (checkBlocked && blockedUsers.indexOf(user) >= 0) {
throw new new Error(util.format(_('Blocked email address "%s"'), address));
}
const result = await new Promise(resolve => {
const result = isemail.validate(address, {
checkDNS: true,
errorLevel: 1
}, resolve);
});
return result;
}
function validateEmailGetMessage(result, address) {
if (result !== 0) {
let message = util.format(_('Invalid email address "%s".'), address);
switch (result) {
case 5:
message += ' ' + _('MX record not found for domain');
break;
case 6:
message += ' ' + _('Address domain not found');
break;
case 12:
message += ' ' + _('Address domain name is required');
break;
}
return message;
}
}

302
lib/tools.js
View file

@ -1,127 +1,97 @@
'use strict';
const config = require('config');
let fs = require('fs');
let path = require('path');
let db = require('./db');
let slugify = require('slugify');
let Isemail = require('isemail');
let urllib = require('url');
let juice = require('juice');
let jsdom = require('jsdom');
let he = require('he');
let _ = require('./translate')._;
let util = require('util');
let createDOMPurify = require('dompurify');
let htmlToText = require('html-to-text');
const _ = require('./translate')._;
const util = require('util');
const isemail = require('isemail');
let blockedUsers = ['abuse', 'admin', 'billing', 'compliance', 'devnull', 'dns', 'ftp', 'hostmaster', 'inoc', 'ispfeedback', 'ispsupport', 'listrequest', 'list', 'maildaemon', 'noc', 'noreply', 'noreply', 'null', 'phish', 'phishing', 'postmaster', 'privacy', 'registrar', 'root', 'security', 'spam', 'support', 'sysadmin', 'tech', 'undisclosedrecipients', 'unsubscribe', 'usenet', 'uucp', 'webmaster', 'www'];
const bluebird = require('bluebird');
module.exports = {
queryParams,
createSlug,
updateMenu,
validateEmail,
formatMessage,
getMessageLinks,
prepareHtml,
purifyHTML,
mergeTemplateIntoLayout,
workers: new Set()
};
const hasher = require('node-object-hash')();
const mjml = require('mjml');
const hbs = require('hbs');
const juice = require('juice');
function queryParams(obj) {
return Object.keys(obj).
filter(key => key !== '_csrf').
map(key => encodeURIComponent(key) + '=' + encodeURIComponent(obj[key])).
join('&');
const fsReadFile = bluebird.promisify(require('fs').readFile);
const jsdomEnv = bluebird.promisify(require('jsdom').env);
const templates = new Map();
async function getTemplate(template) {
if (!template) {
return false;
}
const key = (typeof template === 'object') ? hasher.hash(template) : template;
if (templates.has(key)) {
return templates.get(key);
}
let source;
if (typeof template === 'object') {
source = await mergeTemplateIntoLayout(template.template, template.layout);
} else {
source = await fsReadFile(path.join(__dirname, '..', 'views', template), 'utf-8');
}
if (template.type === 'mjml') {
const compiled = mjml.mjml2html(source);
if (compiled.errors.length) {
throw new Error(compiled.errors[0].message || compiled.errors[0]);
}
source = compiled.html;
}
const renderer = hbs.handlebars.compile(compiled.html);
templates.set(key, renderer);
return renderer;
}
function createSlug(table, name, callback) {
let baseSlug = slugify(name).trim().toLowerCase() || 'list';
let counter = 0;
async function mergeTemplateIntoLayout(template, layout) {
layout = layout || '{{{body}}}';
if (baseSlug.length > 80) {
baseSlug = baseSlug.substr(0, 80);
async function readFile(relPath) {
return await fsReadFile(path.join(__dirname, '..', 'views', relPath), 'utf-8');
}
db.getConnection((err, connection) => {
if (err) {
return callback(err);
// Please dont end your custom messages with .hbs ...
if (layout.endsWith('.hbs')) {
layout = await readFile(layout);
}
let finalize = (err, slug) => {
connection.release();
if (err) {
return callback(err);
if (template.endsWith('.hbs')) {
template = readFile(template);
}
return callback(null, slug);
};
let trySlug = () => {
let currentSlug = baseSlug + (counter === 0 ? '' : '-' + counter);
counter++;
connection.query('SELECT id FROM ' + table + ' WHERE slug=?', [currentSlug], (err, rows) => {
if (err) {
return finalize(err);
}
if (!rows || !rows.length) {
return finalize(null, currentSlug);
}
trySlug();
});
};
trySlug();
});
const source = layout.replace(/\{\{\{body\}\}\}/g, template);
return source;
}
// FIXME - remove once we fully manage the menu in the client
function updateMenu(res) {
if (!res.locals.menu) {
res.locals.menu = [];
}
res.locals.menu.push({
title: _('Lists'),
url: '/lists',
key: 'lists'
}, {
title: _('Templates'),
url: '/templates',
key: 'templates'
}, {
title: _('Campaigns'),
url: '/campaigns',
key: 'campaigns'
}, {
title: _('Automation'),
url: '/triggers',
key: 'triggers'
});
if (config.reports && config.reports.enabled === true) {
res.locals.menu.push({
title: _('Reports'),
url: '/reports',
key: 'reports'
});
}
}
// FIXME - either remove of delegate to validateEmail in tools-async (or vice-versa)
function validateEmail(address, checkBlocked, callback) {
async function validateEmail(address, checkBlocked) {
let user = (address || '').toString().split('@').shift().toLowerCase().replace(/[^a-z0-9]/g, '');
if (checkBlocked && blockedUsers.indexOf(user) >= 0) {
return callback(new Error(util.format(_('Blocked email address "%s"'), address)));
throw new new Error(util.format(_('Blocked email address "%s"'), address));
}
Isemail.validate(address, {
const result = await new Promise(resolve => {
const result = isemail.validate(address, {
checkDNS: true,
errorLevel: 1
}, result => {
}, resolve);
});
return result;
}
function validateEmailGetMessage(result, address) {
if (result !== 0) {
let message = util.format(_('Invalid email address "%s".'), address);
switch (result) {
@ -135,74 +105,26 @@ function validateEmail(address, checkBlocked, callback) {
message += ' ' + _('Address domain name is required');
break;
}
return callback(new Error(message));
return message;
}
return callback();
});
}
function getMessageLinks(serviceUrl, campaign, list, subscription) {
return {
LINK_UNSUBSCRIBE: urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + subscription.cid + '?c=' + campaign.cid),
LINK_PREFERENCES: urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/manage/' + subscription.cid),
LINK_BROWSER: urllib.resolve(serviceUrl, '/archive/' + campaign.cid + '/' + list.cid + '/' + subscription.cid),
CAMPAIGN_ID: campaign.cid,
LIST_ID: list.cid,
SUBSCRIPTION_ID: subscription.cid
};
}
function formatMessage(serviceUrl, campaign, list, subscription, message, filter, isHTML) {
filter = typeof filter === 'function' ? filter : (str => str);
let links = getMessageLinks(serviceUrl, campaign, list, subscription);
let getValue = key => {
key = (key || '').toString().toUpperCase().trim();
if (links.hasOwnProperty(key)) {
return links[key];
}
if (subscription.mergeTags.hasOwnProperty(key)) {
let value = (subscription.mergeTags[key] || '').toString();
let containsHTML = /<[a-z][\s\S]*>/.test(value);
return isHTML ? he.encode((containsHTML ? value : value.replace(/(?:\r\n|\r|\n)/g, '<br/>')), {
useNamedReferences: true,
allowUnsafeSymbols: true
}) : (containsHTML ? htmlToText.fromString(value) : value);
}
return false;
};
return message.replace(/\[([a-z0-9_]+)(?:\/([^\]]+))?\]/ig, (match, identifier, fallback) => {
identifier = identifier.toUpperCase();
let value = getValue(identifier);
if (value === false) {
return match;
}
value = (value || fallback || '').trim();
return filter(value);
});
}
function prepareHtml(html, callback) {
async function prepareHtml(html) {
if (!(html || '').toString().trim()) {
return callback(null, false);
return false;
}
jsdom.env(false, false, {
const win = await jsdomEnv(false, false, {
html,
features: {
FetchExternalResources: false, // disables resource loading over HTTP / filesystem
ProcessExternalResources: false // do not execute JS within script blocks
}
}, (err, win) => {
if (err) {
return callback(err);
}
})
let head = win.document.querySelector('head');
const head = win.document.querySelector('head');
let hasCharsetTag = false;
let metaTags = win.document.querySelectorAll('meta');
const metaTags = win.document.querySelectorAll('meta');
if (metaTags) {
for (let i = 0; i < metaTags.length; i++) {
if (metaTags[i].hasAttribute('charset')) {
@ -213,71 +135,21 @@ function prepareHtml(html, callback) {
}
}
if (!hasCharsetTag) {
let charsetTag = win.document.createElement('meta');
const charsetTag = win.document.createElement('meta');
charsetTag.setAttribute('charset', 'utf-8');
head.appendChild(charsetTag);
}
let preparedHtml = '<!doctype html><html>' + win.document.documentElement.innerHTML + '</html>';
const preparedHtml = '<!doctype html><html>' + win.document.documentElement.innerHTML + '</html>';
return callback(null, juice(preparedHtml));
});
return juice(preparedHtml);
}
function purifyHTML(html) {
let win = jsdom.jsdom('', {
features: {
FetchExternalResources: false, // disables resource loading over HTTP / filesystem
ProcessExternalResources: false // do not execute JS within script blocks
}
}).defaultView;
let DOMPurify = createDOMPurify(win);
return DOMPurify.sanitize(html);
}
// TODO Simplify!
function mergeTemplateIntoLayout(template, layout, callback) {
module.exports = {
validateEmail,
validateEmailGetMessage,
mergeTemplateIntoLayout,
getTemplate,
prepareHtml
};
layout = layout || '{{{body}}}';
let readFile = (relPath, callback) => {
fs.readFile(path.join(__dirname, '..', 'views', relPath), 'utf-8', (err, source) => {
if (err) {
return callback(err);
}
callback(null, source);
});
};
let done = (template, layout) => {
let source = layout.replace(/\{\{\{body\}\}\}/g, template);
return callback(null, source);
};
if (layout.endsWith('.hbs')) {
readFile(layout, (err, layout) => {
if (err) {
return callback(err);
}
// Please dont end your custom messages with .hbs ...
if (template.endsWith('.hbs')) {
readFile(template, (err, template) => {
if (err) {
return callback(err);
}
return done(template, layout);
});
} else {
return done(template, layout);
}
});
} else if (template.endsWith('.hbs')) {
readFile(template, (err, template) => {
if (err) {
return callback(err);
}
return done(template, layout);
});
} else {
return done(template, layout);
}
}

10
lib/urls.js
View file

@ -1,23 +1,23 @@
'use strict';
const config = require('config');
const url = require('url');
const urllib = require('url');
function getTrustedUrl(path) {
return config.www.trustedUrlBase + (path || '');
return urllib.resolve(config.www.trustedUrlBase, path || '');
}
function getSandboxUrl(path) {
return config.www.sandboxUrlBase + (path || '');
return urllib.resolve(config.www.sandboxUrlBase, path || '');
}
function getTrustedUrlBaseDir() {
const mailtrainUrl = url.parse(getTrustedUrl());
const mailtrainUrl = urllib.parse(getTrustedUrl());
return mailtrainUrl.pathname;
}
function getSandboxUrlBaseDir() {
const mailtrainUrl = url.parse(getSandboxUrl());
const mailtrainUrl = urllib.parse(getSandboxUrl());
return mailtrainUrl.pathname;
}

2
models/blacklist.js
View file

@ -3,7 +3,7 @@
const knex = require('../lib/knex');
const dtHelpers = require('../lib/dt-helpers');
const shares = require('./shares');
const tools = require('../lib/tools-async');
const tools = require('../lib/tools');
async function listDTAjax(context, params) {
shares.enforceGlobalPermission(context, 'manageBlacklist');

13
models/send-configuration.js → models/send-configurations.js
View file

@ -8,6 +8,7 @@ const interoperableErrors = require('../shared/interoperable-errors');
const shares = require('./shares');
const namespaceHelpers = require('../lib/namespace-helpers');
const {MailerType, getSystemSendConfigurationId} = require('../shared/send-configurations');
const contextHelpers = require('../lib/context-helpers');
const allowedKeys = new Set(['name', 'description', 'from_email', 'from_email_overridable', 'from_name', 'from_name_overridable', 'subject', 'subject_overridable', 'verp_hostname', 'mailer_type', 'mailer_settings', 'namespace']);
@ -92,6 +93,9 @@ async function updateWithConsistencyCheck(context, entity) {
await shares.rebuildPermissionsTx(tx, { entityTypeId: 'sendConfiguration', entityId: entity.id });
});
// FIXME - recreate respective mailer, notify senders to recreate the mailer
}
async function remove(context, id) {
@ -102,10 +106,16 @@ async function remove(context, id) {
await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'sendConfiguration', id, 'delete');
// FIXME - delete send configuration assignment in campaigns
await tx('lists').update({send_configuration: null}).where('send_configuration', id);
await tx('send_configurations').where('id', id).del();
});
}
async function getSystemSendConfiguration() {
return await getById(contextHelpers.getAdminContext(), getSystemSendConfigurationId());
}
module.exports = {
MailerType,
@ -114,5 +124,6 @@ module.exports = {
getById,
create,
updateWithConsistencyCheck,
remove
remove,
getSystemSendConfiguration
};

2
models/settings.js
View file

@ -51,6 +51,8 @@ async function set(context, data) {
}
}
}
// FIXME - recreate mailers, notify senders to recreate the mailers
}
module.exports = {

14
models/users.js
View file

@ -7,10 +7,10 @@ const { enforce, filterObject } = require('../lib/helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const passwordValidator = require('../shared/password-validator')();
const dtHelpers = require('../lib/dt-helpers');
const tools = require('../lib/tools-async');
const tools = require('../lib/tools');
const crypto = require('crypto');
const settings = require('./settings');
const urllib = require('url');
const {getTrustedUrl} = require('../lib/urls');
const _ = require('../lib/translate')._;
const bluebird = require('bluebird');
@ -19,8 +19,7 @@ const bcrypt = require('bcrypt-nodejs');
const bcryptHash = bluebird.promisify(bcrypt.hash);
const bcryptCompare = bluebird.promisify(bcrypt.compare);
const mailer = require('../lib/mailer');
const mailerSendMail = bluebird.promisify(mailer.sendMail);
const mailers = require('../lib/mailers');
const passport = require('../lib/passport');
@ -301,9 +300,10 @@ async function sendPasswordReset(usernameOrEmail) {
reset_expire: new Date(Date.now() + 60 * 60 * 1000)
});
const { serviceUrl, adminEmail } = await settings.get(['serviceUrl', 'adminEmail']);
const { adminEmail } = await settings.get(contextHelpers.getAdminContext(), ['adminEmail']);
await mailerSendMail({
const mailer = await mailers.getOrCreateMailer();
await mailer.sendMail({
from: {
address: adminEmail
},
@ -318,7 +318,7 @@ async function sendPasswordReset(usernameOrEmail) {
title: 'Mailtrain',
username: user.username,
name: user.name,
confirmUrl: urllib.resolve(serviceUrl, `/account/reset/${encodeURIComponent(user.username)}/${encodeURIComponent(resetToken)}`)
confirmUrl: getTrustedUrl(`/account/reset/${encodeURIComponent(user.username)}/${encodeURIComponent(resetToken)}`)
}
});
}

31
lib/db.js → obsolete/lib/db-old.js
View file

@ -7,6 +7,8 @@ let Lock = require('redfour');
let stringifyDate = require('json-stringify-date');
let senders = require('./senders');
const bluebird = require('bluebird');
module.exports = mysql.createPool(config.mysql);
if (config.redis && config.redis.enabled) {
@ -17,7 +19,7 @@ if (config.redis && config.redis.enabled) {
namespace: 'mailtrain:lock'
});
module.exports.getLock = (id, callback) => {
module.exports.getLock = bluebird.promisify((id, callback) => {
queueLock.waitAcquireLock(id, 60 * 1000 /* Lock expires after 60sec */ , 10 * 1000 /* Wait for lock for up to 10sec */ , (err, lock) => {
if (err) {
return callback(err);
@ -32,13 +34,13 @@ if (config.redis && config.redis.enabled) {
}
});
});
};
});
module.exports.clearCache = (key, callback) => {
module.exports.clearCache = bluebird.promisify((key, callback) => {
module.exports.redis.del('mailtrain:cache:' + key, err => callback(err));
};
});
module.exports.addToCache = (key, value, callback) => {
module.exports.addToCache = bluebird.promisify((key, value, callback) => {
if (!value) {
return setImmediate(() => callback());
}
@ -46,9 +48,9 @@ if (config.redis && config.redis.enabled) {
lpush('mailtrain:cache:' + key, stringifyDate.stringify(value)).
expire('mailtrain:cache:' + key, 24 * 3600).
exec(err => callback(err));
};
});
module.exports.getFromCache = (key, callback) => {
module.exports.getFromCache = bluebird.promisify((key, callback) => {
module.exports.redis.rpop('mailtrain:cache:' + key, (err, value) => {
if (err) {
return callback(err);
@ -61,23 +63,24 @@ if (config.redis && config.redis.enabled) {
return callback(null, value);
});
};
});
} else {
// fakelock. does not lock anything
module.exports.getLock = (id, callback) => {
module.exports.getLock = bluebird.promisify((id, callback) => {
setImmediate(() => callback(null, {
lock: false,
release(done) {
setImmediate(done);
}
}));
};
});
let caches = new Map();
module.exports.clearCache = (key, callback) => {
module.exports.clearCache = bluebird.promisify((key, callback) => {
caches.delete(key);
// This notifies the callback below - i.e. process.on(...) - which is installed in the sender process.
senders.workers.forEach(child => {
child.send({
cmd: 'db.clearCache',
@ -85,7 +88,7 @@ if (config.redis && config.redis.enabled) {
});
});
setImmediate(() => callback());
};
});
process.on('message', m => {
if (m && m.cmd === 'db.clearCache' && m.key) {
@ -93,13 +96,13 @@ if (config.redis && config.redis.enabled) {
}
});
module.exports.addToCache = (key, value, callback) => {
module.exports.addToCache = bluebird.promisify((key, value, callback) => {
if (!caches.has(key)) {
caches.set(key, []);
}
caches.get(key).push(value);
setImmediate(() => callback());
};
});
module.exports.getFromCache = (key, callback) => {
let value;

0
lib/editor-helpers.js → obsolete/lib/editor-helpers.js
View file

12
lib/mailer.js → obsolete/lib/mailer-old.js
View file

@ -35,10 +35,20 @@ Handlebars.registerHelper('translate', function (context, options) { // eslint-d
return new Handlebars.SafeString(result);
});
async function getTransport(sendConfiguration) {
if (!sendConfiguration) {
}
}
module.exports.transport = false;
module.exports.update = () => {
createMailer(() => false);
createMailer();
};
module.exports.getMailer = callback => {

0
lib/models/blacklist.js → obsolete/lib/models/blacklist.js
View file

0
lib/models/campaigns.js → obsolete/lib/models/campaigns.js
View file

0
lib/models/confirmations.js → obsolete/lib/models/confirmations.js
View file

0
lib/models/fields.js → obsolete/lib/models/fields.js
View file

0
lib/models/forms.js → obsolete/lib/models/forms.js
View file

0
lib/models/links.js → obsolete/lib/models/links.js
View file

0
lib/models/lists.js → obsolete/lib/models/lists.js
View file

0
lib/models/segments.js → obsolete/lib/models/segments.js
View file

0
lib/models/subscriptions.js → obsolete/lib/models/subscriptions.js
View file

0
lib/models/templates.js → obsolete/lib/models/templates.js
View file

0
lib/models/triggers.js → obsolete/lib/models/triggers.js
View file

0
lib/table-helpers.js → obsolete/lib/table-helpers-old.js
View file

283
obsolete/lib/tools-old.js Normal file
View file

@ -0,0 +1,283 @@
'use strict';
const config = require('config');
let fs = require('fs');
let path = require('path');
let db = require('./db');
let slugify = require('slugify');
let Isemail = require('isemail');
let urllib = require('url');
let juice = require('juice');
let jsdom = require('jsdom');
let he = require('he');
let _ = require('./translate')._;
let util = require('util');
let createDOMPurify = require('dompurify');
let htmlToText = require('html-to-text');
let blockedUsers = ['abuse', 'admin', 'billing', 'compliance', 'devnull', 'dns', 'ftp', 'hostmaster', 'inoc', 'ispfeedback', 'ispsupport', 'listrequest', 'list', 'maildaemon', 'noc', 'noreply', 'noreply', 'null', 'phish', 'phishing', 'postmaster', 'privacy', 'registrar', 'root', 'security', 'spam', 'support', 'sysadmin', 'tech', 'undisclosedrecipients', 'unsubscribe', 'usenet', 'uucp', 'webmaster', 'www'];
module.exports = {
queryParams,
createSlug,
updateMenu,
validateEmail,
formatMessage,
getMessageLinks,
prepareHtml,
purifyHTML,
mergeTemplateIntoLayout,
workers: new Set()
};
function queryParams(obj) {
return Object.keys(obj).
filter(key => key !== '_csrf').
map(key => encodeURIComponent(key) + '=' + encodeURIComponent(obj[key])).
join('&');
}
function createSlug(table, name, callback) {
let baseSlug = slugify(name).trim().toLowerCase() || 'list';
let counter = 0;
if (baseSlug.length > 80) {
baseSlug = baseSlug.substr(0, 80);
}
db.getConnection((err, connection) => {
if (err) {
return callback(err);
}
let finalize = (err, slug) => {
connection.release();
if (err) {
return callback(err);
}
return callback(null, slug);
};
let trySlug = () => {
let currentSlug = baseSlug + (counter === 0 ? '' : '-' + counter);
counter++;
connection.query('SELECT id FROM ' + table + ' WHERE slug=?', [currentSlug], (err, rows) => {
if (err) {
return finalize(err);
}
if (!rows || !rows.length) {
return finalize(null, currentSlug);
}
trySlug();
});
};
trySlug();
});
}
// FIXME - remove once we fully manage the menu in the client
function updateMenu(res) {
if (!res.locals.menu) {
res.locals.menu = [];
}
res.locals.menu.push({
title: _('Lists'),
url: '/lists',
key: 'lists'
}, {
title: _('Templates'),
url: '/templates',
key: 'templates'
}, {
title: _('Campaigns'),
url: '/campaigns',
key: 'campaigns'
}, {
title: _('Automation'),
url: '/triggers',
key: 'triggers'
});
if (config.reports && config.reports.enabled === true) {
res.locals.menu.push({
title: _('Reports'),
url: '/reports',
key: 'reports'
});
}
}
// FIXME - either remove of delegate to validateEmail in tools-async (or vice-versa)
function validateEmail(address, checkBlocked, callback) {
let user = (address || '').toString().split('@').shift().toLowerCase().replace(/[^a-z0-9]/g, '');
if (checkBlocked && blockedUsers.indexOf(user) >= 0) {
return callback(new Error(util.format(_('Blocked email address "%s"'), address)));
}
Isemail.validate(address, {
checkDNS: true,
errorLevel: 1
}, result => {
if (result !== 0) {
let message = util.format(_('Invalid email address "%s".'), address);
switch (result) {
case 5:
message += ' ' + _('MX record not found for domain');
break;
case 6:
message += ' ' + _('Address domain not found');
break;
case 12:
message += ' ' + _('Address domain name is required');
break;
}
return callback(new Error(message));
}
return callback();
});
}
function getMessageLinks(serviceUrl, campaign, list, subscription) {
return {
LINK_UNSUBSCRIBE: urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/unsubscribe/' + subscription.cid + '?c=' + campaign.cid),
LINK_PREFERENCES: urllib.resolve(serviceUrl, '/subscription/' + list.cid + '/manage/' + subscription.cid),
LINK_BROWSER: urllib.resolve(serviceUrl, '/archive/' + campaign.cid + '/' + list.cid + '/' + subscription.cid),
CAMPAIGN_ID: campaign.cid,
LIST_ID: list.cid,
SUBSCRIPTION_ID: subscription.cid
};
}
function formatMessage(serviceUrl, campaign, list, subscription, message, filter, isHTML) {
filter = typeof filter === 'function' ? filter : (str => str);
let links = getMessageLinks(serviceUrl, campaign, list, subscription);
let getValue = key => {
key = (key || '').toString().toUpperCase().trim();
if (links.hasOwnProperty(key)) {
return links[key];
}
if (subscription.mergeTags.hasOwnProperty(key)) {
let value = (subscription.mergeTags[key] || '').toString();
let containsHTML = /<[a-z][\s\S]*>/.test(value);
return isHTML ? he.encode((containsHTML ? value : value.replace(/(?:\r\n|\r|\n)/g, '<br/>')), {
useNamedReferences: true,
allowUnsafeSymbols: true
}) : (containsHTML ? htmlToText.fromString(value) : value);
}
return false;
};
return message.replace(/\[([a-z0-9_]+)(?:\/([^\]]+))?\]/ig, (match, identifier, fallback) => {
identifier = identifier.toUpperCase();
let value = getValue(identifier);
if (value === false) {
return match;
}
value = (value || fallback || '').trim();
return filter(value);
});
}
function prepareHtml(html, callback) {
if (!(html || '').toString().trim()) {
return callback(null, false);
}
jsdom.env(false, false, {
html,
features: {
FetchExternalResources: false, // disables resource loading over HTTP / filesystem
ProcessExternalResources: false // do not execute JS within script blocks
}
}, (err, win) => {
if (err) {
return callback(err);
}
let head = win.document.querySelector('head');
let hasCharsetTag = false;
let metaTags = win.document.querySelectorAll('meta');
if (metaTags) {
for (let i = 0; i < metaTags.length; i++) {
if (metaTags[i].hasAttribute('charset')) {
metaTags[i].setAttribute('charset', 'utf-8');
hasCharsetTag = true;
break;
}
}
}
if (!hasCharsetTag) {
let charsetTag = win.document.createElement('meta');
charsetTag.setAttribute('charset', 'utf-8');
head.appendChild(charsetTag);
}
let preparedHtml = '<!doctype html><html>' + win.document.documentElement.innerHTML + '</html>';
return callback(null, juice(preparedHtml));
});
}
function purifyHTML(html) {
let win = jsdom.jsdom('', {
features: {
FetchExternalResources: false, // disables resource loading over HTTP / filesystem
ProcessExternalResources: false // do not execute JS within script blocks
}
}).defaultView;
let DOMPurify = createDOMPurify(win);
return DOMPurify.sanitize(html);
}
// TODO Simplify!
function mergeTemplateIntoLayout(template, layout, callback) {
layout = layout || '{{{body}}}';
let readFile = (relPath, callback) => {
fs.readFile(path.join(__dirname, '..', 'views', relPath), 'utf-8', (err, source) => {
if (err) {
return callback(err);
}
callback(null, source);
});
};
let done = (template, layout) => {
let source = layout.replace(/\{\{\{body\}\}\}/g, template);
return callback(null, source);
};
if (layout.endsWith('.hbs')) {
readFile(layout, (err, layout) => {
if (err) {
return callback(err);
}
// Please dont end your custom messages with .hbs ...
if (template.endsWith('.hbs')) {
readFile(template, (err, template) => {
if (err) {
return callback(err);
}
return done(template, layout);
});
} else {
return done(template, layout);
}
});
} else if (template.endsWith('.hbs')) {
readFile(template, (err, template) => {
if (err) {
return callback(err);
}
return done(template, layout);
});
} else {
return done(template, layout);
}
}

2
routes/api.js
View file

@ -1,7 +1,7 @@
'use strict';
const lists = require('../models/lists');
const tools = require('../lib/tools-async');
const tools = require('../lib/tools');
const blacklist = require('../models/blacklist');
const fields = require('../models/fields');
const { SubscriptionStatus } = require('../shared/lists');

1
routes/rest/editors.js
View file

@ -9,6 +9,7 @@ const premailerPrepareAsync = bluebird.promisify(premailerApi.prepare);
const router = require('../../lib/router-async').create();
/*
FIXME
const { nodeifyFunction } = require('../lib/nodeify');
const getSettings = nodeifyFunction(require('../models/settings').get);

2
routes/rest/send-configurations.js
View file

@ -1,7 +1,7 @@
'use strict';
const passport = require('../../lib/passport');
const sendConfigurations = require('../../models/send-configuration');
const sendConfigurations = require('../../models/send-configurations');
const router = require('../../lib/router-async').create();

97
routes/subscription.js
View file

@ -12,31 +12,21 @@ const settings = require('../models/settings');
const _ = require('../lib/translate')._;
const contextHelpers = require('../lib/context-helpers');
const forms = require('../models/forms');
const {getTrustedUrl} = require('../lib/urls');
const { SubscriptionStatus } = require('../shared/lists');
const openpgp = require('openpgp');
const util = require('util');
const cors = require('cors');
const cache = require('memory-cache');
const geoip = require('geoip-ultralight');
const passport = require('../lib/passport');
const tools = require('../lib/tools-async');
const helpers = require('../lib/helpers');
const tools = require('../lib/tools');
const mailHelpers = require('../lib/subscription-mail-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const mjml = require('mjml');
const hbs = require('hbs');
const mjmlTemplates = new Map();
const objectHash = require('object-hash');
const bluebird = require('bluebird');
const fsReadFile = bluebird.promisify(require('fs').readFile);
const { cleanupFromPost } = require('../lib/helpers');
const originWhitelist = config.cors && config.cors.origins || [];
@ -87,7 +77,7 @@ async function injectCustomFormData(customFormId, viewKey, data) {
}
if (!customFormId) {
data.formInputStyle = '@import url(/subscription/form-input-style.css);';
data.formInputStyle = '@import url(/public/subscription/form-input-style.css);';
return;
}
@ -95,40 +85,14 @@ async function injectCustomFormData(customFormId, viewKey, data) {
data.template.template = form[viewKey] || data.template.template;
data.template.layout = form.layout || data.template.layout;
data.formInputStyle = form.formInputStyle || '@import url(/subscription/form-input-style.css);';
data.formInputStyle = form.formInputStyle || '@import url(/public/subscription/form-input-style.css);';
const configItems = await settings.get(['uaCode']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['uaCode']);
data.uaCode = configItems.uaCode;
data.customSubscriptionScripts = config.customSubscriptionScripts || [];
}
async function getMjmlTemplate(template) {
let key = (typeof template === 'object') ? objectHash(template) : template;
if (mjmlTemplates.has(key)) {
return mjmlTemplates.get(key);
}
let source;
if (typeof template === 'object') {
source = await tools.mergeTemplateIntoLayout(template.template, template.layout);
} else {
source = await fsReadFile(path.join(__dirname, '..', 'views', template), 'utf-8');
}
const compiled = mjml.mjml2html(source);
if (compiled.errors.length) {
throw new Error(compiled.errors[0].message || compiled.errors[0]);
}
const renderer = hbs.handlebars.compile(compiled.html);
mjmlTemplates.set(key, renderer);
return renderer;
}
async function captureFlashMessages(res) {
const renderAsync = bluebird.promisify(res.render.bind(res));
return await renderAsync('subscription/capture-flash-messages', { layout: null });
@ -205,18 +169,18 @@ async function _renderSubscribe(req, res, list, subscription) {
data.customFields = await fields.forHbs(contextHelpers.getAdminContext(), list.id, subscription);
data.useEditor = true;
const configItems = await settings.get(['pgpPrivateKey', 'defaultAddress']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey']);
data.hasPubkey = !!configItems.pgpPrivateKey;
data.defaultAddress = configItems.defaultAddress;
data.template = {
template: 'subscription/web-subscribe.mjml.hbs',
layout: 'subscription/layout.mjml.hbs'
layout: 'subscription/layout.mjml.hbs',
type: 'mjml'
};
await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-subscribe', data);
const htmlRenderer = await getMjmlTemplate(data.template);
const htmlRenderer = await tools.getTemplate(data.template);
data.isWeb = true;
data.needsJsWarning = true;
@ -354,12 +318,13 @@ router.getAsync('/:cid/widget', cors(corsOptions), async (req, res) => {
const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
const configItems = await settings.get(['serviceUrl', 'pgpPrivateKey']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey']);
const data = {
title: list.name,
cid: list.cid,
serviceUrl: configItems.serviceUrl,
publicKeyUrl: getTrustedUrl('subscription/publickey'),
subscribeUrl: getTrustedUrl(`subscription/${list.cid}/subscribe`),
hasPubkey: !!configItems.pgpPrivateKey,
customFields: await fields.forHbs(contextHelpers.getAdminContext(), list.id),
template: {},
@ -406,18 +371,18 @@ router.getAsync('/:lcid/manage/:ucid', passport.csrfProtection, async (req, res)
data.useEditor = true;
const configItems = await settings.get(['pgpPrivateKey', 'defaultAddress']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPrivateKey']);
data.hasPubkey = !!configItems.pgpPrivateKey;
data.defaultAddress = configItems.defaultAddress;
data.template = {
template: 'subscription/web-manage.mjml.hbs',
layout: 'subscription/layout.mjml.hbs'
layout: 'subscription/layout.mjml.hbs',
type: 'mjml'
};
await injectCustomFormData(req.query.fid || list.default_form, 'data/web-manage', data);
const htmlRenderer = await getMjmlTemplate(data.template);
const htmlRenderer = await tools.getTemplate(data.template);
data.isWeb = true;
data.needsJsWarning = true;
@ -452,24 +417,22 @@ router.getAsync('/:lcid/manage-address/:ucid', passport.csrfProtection, async (r
throw new interoperableErrors.NotFoundError('Subscription not found in this list');
}
const configItems = await settings.get(['defaultAddress']);
const data = {};
data.email = subscription.email;
data.cid = subscription.cid;
data.lcid = req.params.lcid;
data.title = list.name;
data.csrfToken = req.csrfToken();
data.defaultAddress = configItems.defaultAddress;
data.template = {
template: 'subscription/web-manage-address.mjml.hbs',
layout: 'subscription/layout.mjml.hbs'
layout: 'subscription/layout.mjml.hbs',
type: 'mjml'
};
await injectCustomFormData(req.query.fid || list.default_form, 'data/web-manage-address', data);
const htmlRenderer = await getMjmlTemplate(data.template);
const htmlRenderer = await tools.getTemplate(data.template);
data.isWeb = true;
data.needsJsWarning = true;
@ -535,7 +498,7 @@ router.postAsync('/:lcid/manage-address', passport.parseForm, passport.csrfProte
router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req, res) => {
const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.lcid);
const configItems = await settings.get(['defaultAddress']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['defaultAddress']);
const autoUnsubscribe = req.query.auto === 'yes';
@ -563,12 +526,13 @@ router.getAsync('/:lcid/unsubscribe/:ucid', passport.csrfProtection, async (req,
data.template = {
template: 'subscription/web-unsubscribe.mjml.hbs',
layout: 'subscription/layout.mjml.hbs'
layout: 'subscription/layout.mjml.hbs',
type: 'mjml'
};
await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-unsubscribe', data);
const htmlRenderer = await getMjmlTemplate(data.template);
const htmlRenderer = await tools.getTemplate(data.template);
data.isWeb = true;
data.needsJsWarning = true;
@ -660,7 +624,7 @@ router.getAsync('/:cid/manual-unsubscribe-notice', async (req, res) => {
});
router.postAsync('/publickey', passport.parseForm, async (req, res) => {
const configItems = await settings.get(['pgpPassphrase', 'pgpPrivateKey']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['pgpPassphrase', 'pgpPrivateKey']);
if (!configItems.pgpPrivateKey) {
const err = new Error(_('Public key is not set'));
@ -698,23 +662,22 @@ router.postAsync('/publickey', passport.parseForm, async (req, res) => {
async function webNotice(type, req, res) {
const list = await lists.getByCid(contextHelpers.getAdminContext(), req.params.cid);
const configItems = await settings.get(['defaultHomepage', 'serviceUrl', 'defaultAddress', 'adminEmail']);
const configItems = await settings.get(contextHelpers.getAdminContext(), ['defaultHomepage', 'adminEmail']);
const data = {
title: list.name,
homepage: configItems.defaultHomepage || configItems.serviceUrl,
defaultAddress: configItems.defaultAddress,
contactAddress: configItems.defaultAddress,
homepage: configItems.defaultHomepage || getTrustedUrl(),
contactAddress: list.from_email || configItems.adminEmail,
template: {
template: 'subscription/web-' + type + '-notice.mjml.hbs',
layout: 'subscription/layout.mjml.hbs'
layout: 'subscription/layout.mjml.hbs',
type: 'mjml'
}
};
await injectCustomFormData(req.query.fid || list.default_form, 'subscription/web-' + type + '-notice', data);
const htmlRenderer = await getMjmlTemplate(data.template);
const htmlRenderer = await tools.getTemplate(data.template);
data.isWeb = true;
data.isConfirmNotice = true; // FIXME: Not sure what this does. Check it in a browser with disabled JS

2
services/feedcheck.js
View file

@ -1,5 +1,7 @@
'use strict';
// FIXME - revisit and rewrite if necessary
let log = require('npmlog');
let db = require('../lib/db');

2
services/importer.js
View file

@ -1,5 +1,7 @@
'use strict';
// FIXME - revisit and rewrite if necessary
let log = require('npmlog');
let db = require('../lib/db');

14
services/postfix-bounce-server.js
View file

@ -1,13 +1,15 @@
'use strict';
let log = require('npmlog');
let config = require('config');
let net = require('net');
let campaigns = require('../lib/models/campaigns');
// FIXME - port for the new campaigns model
let seenIds = new Set();
const log = require('npmlog');
const config = require('config');
const net = require('net');
const campaigns = require('../lib/models/campaigns');
let server = net.createServer(socket => {
const seenIds = new Set();
const server = net.createServer(socket => {
let remainder = '';
let reading = false;

22
services/sender.js
View file

@ -1,5 +1,7 @@
'use strict';
// FIXME - update/rewrite
const { nodeifyFunction } = require('../lib/nodeify');
const getSettings = nodeifyFunction(require('../models/settings').get);
@ -7,7 +9,7 @@ let log = require('npmlog');
let config = require('config');
let db = require('../lib/db');
let tools = require('../lib/tools');
let mailer = require('../lib/mailer');
let mailer = require('../lib/mailers');
let campaigns = require('../lib/models/campaigns');
let segments = require('../lib/models/segments');
let lists = require('../lib/models/lists');
@ -462,7 +464,7 @@ function formatMessage(message, callback) {
}
let sendLoop = () => {
mailer.getMailer(err => {
mailers.getMailer(err => {
if (err) {
log.error('Mail', err.stack);
return setTimeout(sendLoop, 10 * 1000);
@ -471,14 +473,14 @@ let sendLoop = () => {
let isThrottled = false;
let getNext = () => {
if (!mailer.transport.isIdle() || isThrottled) {
// only retrieve new messages if there are free slots in the mailer queue
if (!mailers.transport.isIdle() || isThrottled) {
// only retrieve new messages if there are free slots in the mailers queue
return;
}
isThrottled = true;
mailer.transport.checkThrottling(() => {
mailers.transport.checkThrottling(() => {
isThrottled = false;
@ -495,7 +497,7 @@ let sendLoop = () => {
}
// log.verbose('Mail', 'Found new message to be delivered: %s', message.subscription.cid);
// format message to nodemailer message format
// format message to nodemailers message format
formatMessage(message, (err, mail) => {
if (err) {
log.error('Mail', err.stack);
@ -516,7 +518,7 @@ let sendLoop = () => {
tryCount++;
// send the message
mailer.transport.sendMail(mail, (err, info) => {
mailers.transport.sendMail(mail, (err, info) => {
if (err) {
log.error('Mail', err.stack);
if (err.responseCode && err.responseCode >= 400 && err.responseCode < 500 && tryCount <= 5) {
@ -589,7 +591,7 @@ let sendLoop = () => {
});
};
mailer.transport.on('idle', getNext);
mailers.transport.on('idle', getNext);
setImmediate(getNext);
});
};
@ -598,7 +600,7 @@ sendLoop();
process.on('message', m => {
if (m && m.reload) {
log.info('Sender/' + process.pid, 'Reloading mailer config');
mailer.update();
log.info('Sender/' + process.pid, 'Reloading mailers config');
mailers.update();
}
});

18
services/test-server.js
View file

@ -1,18 +1,18 @@
'use strict';
let log = require('npmlog');
let config = require('config');
let crypto = require('crypto');
let humanize = require('humanize');
let http = require('http');
const log = require('npmlog');
const config = require('config');
const crypto = require('crypto');
const humanize = require('humanize');
const http = require('http');
let SMTPServer = require('smtp-server').SMTPServer;
let simpleParser = require('mailparser').simpleParser;
const SMTPServer = require('smtp-server').SMTPServer;
const simpleParser = require('mailparser').simpleParser;
let totalMessages = 0;
let received = 0;
let mailstore = {
const mailstore = {
accounts: {},
saveMessage(address, message) {
if (!this.accounts[address]) {
@ -36,7 +36,7 @@ let mailstore = {
};
// Setup server
let server = new SMTPServer({
const server = new SMTPServer({
// log to console
logger: config.testserver.logger,

2
services/triggers.js
View file

@ -1,5 +1,7 @@
'use strict';
// FIXME - revisit and rewrite if necessary
let log = require('npmlog');
let db = require('../lib/db');
let tools = require('../lib/tools');

63
services/tzupdate.js
View file

@ -8,58 +8,47 @@
// JOIN with subscription table. Subscription table includes timezone name for
// a subscriber and tzoffset table includes offset from UTC in minutes
let moment = require('moment-timezone');
let db = require('../lib/db');
let log = require('npmlog');
const moment = require('moment-timezone');
const knex = require('../lib/knex');
const log = require('npmlog');
let lastCheck = false;
const timezone_timeout = 60 * 60 * 1000;
function updateTimezoneOffsets(callback) {
async function updateTimezoneOffsets() {
log.verbose('UTC', 'Updating timezone offsets');
db.getConnection((err, connection) => {
if (err) {
return callback(err);
const values = [];
for (const tz of moment.tz.names()) {
values.push({
tz: tz.toLowerCase().trim(),
offset: moment.tz(tz).utcOffset()
});
}
let values = [];
moment.tz.names().forEach(tz => {
let time = moment();
values.push('(' + connection.escape(tz.toLowerCase().trim()) + ',' + connection.escape(time.tz(tz).utcOffset()) + ')');
});
let query = 'INSERT INTO tzoffset (`tz`, `offset`) VALUES ' + values.join(', ') + ' ON DUPLICATE KEY UPDATE `offset` = VALUES(`offset`)';
connection.query(query, values, (err, result) => {
connection.release();
if (err) {
return callback(err);
}
return callback(null, result);
});
await knex.transaction(async tx => {
await tx('tzoffset').del();
await tx('tzoffset').insert(values);
});
}
module.exports = callback => {
updateTimezoneOffsets(err => {
if (err) {
return callback(err);
}
let checkLoop = () => {
function start() {
let curUtcDate = new Date().toISOString().split('T').shift();
if (curUtcDate !== lastCheck) {
updateTimezoneOffsets(err => {
if (err) {
updateTimezoneOffsets()
.then(() => {
setTimeout(start, timezone_timeout)
})
.catch(err => {
log.error('UTC', err);
}
setTimeout(checkLoop, timezone_timeout);
setTimeout(start, timezone_timeout);
});
} else {
setTimeout(checkLoop, timezone_timeout);
setTimeout(start, timezone_timeout);
}
lastCheck = curUtcDate;
};
setTimeout(checkLoop, timezone_timeout);
callback(null, true);
});
}
module.exports = {
start
};

2
services/verp-server.js
View file

@ -1,5 +1,7 @@
'use strict';
// FIXME - port for the new campaigns model
const { nodeifyFunction } = require('../lib/nodeify');
const getSettings = nodeifyFunction(require('../models/settings').get);

3
setup/knex/migrations/20180414120444_transform_settings.js
View file

@ -37,6 +37,7 @@ exports.up = (knex, Promise) => (async() => {
await knex.schema.table('lists', table => {
table.string('contact_email');
table.string('homepage');
table.integer('send_configuration').unsigned().references(`send_configurations.id`);
});
const settingsRows = await knex('settings').select(['key', 'value']);
@ -101,6 +102,8 @@ exports.up = (knex, Promise) => (async() => {
namespace: getGlobalNamespaceId()
});
await knex('lists').update({send_configuration: getSystemSendConfigurationId()});
await knex('settings').del();
await knex('settings').insert([
{ key: 'uaCode', value: settings.uaCode },

4
views/subscription/partials/subscription-subscribe-form.hbs
View file

@ -21,5 +21,5 @@
document.getElementById('sub').value = new Date().getTime();
</script>
<script src="/moment/moment.min.js"></script>
<script src="/moment/moment-timezone-with-data.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.12.0/moment.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment-timezone/0.5.3/moment-timezone-with-data.min.js"></script>

4
views/subscription/widget-subscribe.hbs
View file

@ -1,13 +1,13 @@
<div id="mailtrain-subscription-widget-{{cid}}" class="mailtrain-subscription-widget">
{{#if hasPubkey}}
<form method="post" id="download-pubkey" action="{{serviceUrl}}subscription/publickey" style="display: none;">
<form method="post" id="download-pubkey" action="{{publicKeyUrl}}" style="display: none;">
<input type="hidden" name="cid" value="{{cid}}">
</form>
{{/if}}
<h3>{{title}}</h3>
<form class="form" method="post" action="{{serviceUrl}}subscription/{{cid}}/subscribe">
<form class="form" method="post" action="{{subscribeUrl}}">
<input type="hidden" class="tz-detect" name="tz" value="">
<input type="hidden" name="address" value="">
<input type="hidden" class="sub-time" name="sub" value="">