From 69ce80ebfd9fb91d3f7adced06c1b08f0ccdfe89 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Sat, 10 Aug 2019 23:15:38 +0200 Subject: [PATCH 1/9] Fix for #663. Unfortunately, the migration 20190726150000_shorten_field_column_names.js corrupted the segments table. There is no automatic fix. If this affected you, you have to either revert the DB or fix the segments manually. --- .../migrations/20170506102634_v1_to_v2.js | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/server/setup/knex/migrations/20170506102634_v1_to_v2.js b/server/setup/knex/migrations/20170506102634_v1_to_v2.js index d64a40c6..f698e4d3 100644 --- a/server/setup/knex/migrations/20170506102634_v1_to_v2.js +++ b/server/setup/knex/migrations/20170506102634_v1_to_v2.js @@ -11,6 +11,8 @@ const { EntityVals: TriggerEntityVals, EventVals: TriggerEventVals } = require(' const { SubscriptionSource } = require('../../../../shared/lists'); const {DOMParser, XMLSerializer} = require('xmldom'); const log = require('../../../lib/log'); +const shortid = require('shortid'); +const slugify = require('slugify'); const entityTypesAddNamespace = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'user']; const shareableEntityTypes = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'namespace', 'send_configuration', 'mosaico_template']; @@ -236,16 +238,54 @@ async function migrateUsers(knex) { }); } +async function shortenFieldColumnNames(knex, list) { + const fields = await knex('custom_fields').whereNotNull('column').where('list', list.id); + + const fieldsMap = new Map(); + + for (const field of fields) { + const oldName = field.column; + const newName = ('custom_' + slugify(field.name, '_').substring(0,32) + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '_'); + + fieldsMap.set(oldName, newName); + + await knex('custom_fields').where('id', field.id).update('column', newName); + + await knex.schema.table('subscription__' + list.id, table => { + table.renameColumn(oldName, newName); + }); + } + + + function processRule(rule) { + if (rule.type === 'all' || rule.type === 'some' || rule.type === 'none') { + for (const childRule of rule.rules) { + processRule(childRule); + } + } else { + rule.column = fieldsMap.get(rule.column) || rule.column /* this is to handle "email" column */; + } + } + + const segments = await knex('segments').where('list', list.id); + for (const segment of segments) { + const settings = JSON.parse(segment.settings); + processRule(settings.rootRule); + await knex('segments').where('id', segment.id).update({settings: JSON.stringify(settings)}); + } +} + async function migrateSubscriptions(knex) { await knex.schema.dropTableIfExists('subscription'); const lists = await knex('lists'); for (const list of lists) { + await shortenFieldColumnNames(knex, list); + await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `unsubscribed` timestamp NULL DEFAULT NULL'); await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `source_email` int(11) DEFAULT NULL'); await knex.schema.raw('ALTER TABLE `subscription__' + list.id + '` ADD `hash_email` varchar(255) CHARACTER SET ascii'); - const fields = await knex('custom_fields').where('list', list.id); const info = await knex('subscription__' + list.id).columnInfo(); for (const field of fields) { @@ -1251,11 +1291,12 @@ exports.up = (knex, Promise) => (async() => { await migrateCustomFields(knex); log.verbose('Migration', 'Custom fields complete') - await migrateSubscriptions(knex); - await migrateSegments(knex); log.verbose('Migration', 'Segments complete') + await migrateSubscriptions(knex); + log.verbose('Migration', 'Subscriptions complete') + await migrateReports(knex); log.verbose('Migration', 'Reports complete') From 588cf3481037a2519cebc76c9e1827481e510822 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Sat, 10 Aug 2019 23:17:15 +0200 Subject: [PATCH 2/9] Fix for #663 --- ...190726150000_shorten_field_column_names.js | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 server/setup/knex/migrations/20190726150000_shorten_field_column_names.js diff --git a/server/setup/knex/migrations/20190726150000_shorten_field_column_names.js b/server/setup/knex/migrations/20190726150000_shorten_field_column_names.js deleted file mode 100644 index c1fc42eb..00000000 --- a/server/setup/knex/migrations/20190726150000_shorten_field_column_names.js +++ /dev/null @@ -1,21 +0,0 @@ -const shortid = require('shortid'); - -exports.up = (knex, Promise) => (async() => { - const fields = await knex('custom_fields').whereNotNull('column'); - - for (const field of fields) { - const listId = field.list; - const oldName = field.column; - const newName = ('custom_' + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '_'); - - await knex('custom_fields').where('id', field.id).update('column', newName); - - await knex.schema.table('subscription__' + listId, table => { - table.renameColumn(oldName, newName); - table.renameColumn('source_' + oldName, 'source_' + newName); - }); - } -})(); - -exports.down = (knex, Promise) => (async() => { -})(); From 8cb24feca116b43979e73184e9efe491262f4410 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Sun, 11 Aug 2019 16:28:11 +0200 Subject: [PATCH 3/9] Fix for #660 Campaign preview and campaign test send pulls the first entry in the RSS feed and substitutes its data in `[RSS_ENTRY_*]` --- client/src/campaigns/Status.js | 25 +++++++++-- client/src/lib/urls.js | 10 ++++- client/src/templates/helpers.js | 8 ++++ server/app-builder.js | 17 ++++++- server/lib/feedcheck.js | 78 ++++++++++++++++++++++++++++++++- server/lib/message-sender.js | 29 +++++++----- server/models/campaigns.js | 44 ++++++++++++++++++- server/models/subscriptions.js | 2 +- server/routes/campaigns.js | 69 +++++++++++++++++++++++++++++ server/routes/subscriptions.js | 2 - server/services/feedcheck.js | 35 +-------------- 11 files changed, 262 insertions(+), 57 deletions(-) create mode 100644 server/routes/campaigns.js diff --git a/client/src/campaigns/Status.js b/client/src/campaigns/Status.js index c20c7486..6cfd8b92 100644 --- a/client/src/campaigns/Status.js +++ b/client/src/campaigns/Status.js @@ -10,7 +10,7 @@ import {getCampaignLabels} from './helpers'; import {Table} from "../lib/table"; import {Button, Icon, ModalDialog} from "../lib/bootstrap-components"; import axios from "../lib/axios"; -import {getPublicUrl, getUrl} from "../lib/urls"; +import {getPublicUrl, getSandboxUrl, getUrl} from "../lib/urls"; import interoperableErrors from '../../../shared/interoperable-errors'; import {CampaignStatus, CampaignType} from "../../../shared/campaigns"; import moment from 'moment-timezone'; @@ -58,10 +58,29 @@ class PreviewForTestUserModalDialog extends Component { async previewAsync() { if (this.isFormWithoutErrors()) { - const campaignCid = this.props.entity.cid; + const entity = this.props.entity; + const campaignCid = entity.cid; const [listCid, subscriptionCid] = this.getFormValue('testUser').split(':'); - window.open(getPublicUrl(`archive/${campaignCid}/${listCid}/${subscriptionCid}`, {withLocale: true}), '_blank'); + if (entity.type === CampaignType.RSS) { + const result = await axios.post(getUrl('rest/restricted-access-token'), { + method: 'rssPreview', + params: { + campaignCid, + listCid + } + }); + + const accessToken = result.data; + window.open(getSandboxUrl(`campaigns/rss-preview/${campaignCid}/${listCid}/${subscriptionCid}`, accessToken, {withLocale: true}), '_blank'); + + } else if (entity.type === CampaignType.REGULAR) { + window.open(getPublicUrl(`archive/${campaignCid}/${listCid}/${subscriptionCid}`, {withLocale: true}), '_blank'); + + } else { + throw new Error('Preview not supported'); + } + } else { this.showFormValidation(); } diff --git a/client/src/lib/urls.js b/client/src/lib/urls.js index 242a2eaa..b8c5270f 100644 --- a/client/src/lib/urls.js +++ b/client/src/lib/urls.js @@ -15,9 +15,15 @@ function getTrustedUrl(path) { return mailtrainConfig.trustedUrlBase + (path || ''); } -function getSandboxUrl(path, customRestrictedAccessToken) { +function getSandboxUrl(path, customRestrictedAccessToken, opts) { const localRestrictedAccessToken = customRestrictedAccessToken || restrictedAccessToken; - return mailtrainConfig.sandboxUrlBase + localRestrictedAccessToken + '/' + (path || ''); + const url = new URL(localRestrictedAccessToken + '/' + (path || ''), mailtrainConfig.sandboxUrlBase); + + if (opts && opts.withLocale) { + url.searchParams.append('locale', i18n.language); + } + + return url.toString(); } function getPublicUrl(path, opts) { diff --git a/client/src/templates/helpers.js b/client/src/templates/helpers.js index 52d7a319..dd651149 100644 --- a/client/src/templates/helpers.js +++ b/client/src/templates/helpers.js @@ -683,6 +683,14 @@ export function getEditForm(owner, typeKey, prefix = '') { RSS entry image URL + + + {tg('RSS_ENTRY_CUSTOM_TAGS')} + + + Mailtrain custom tags. The custom tags can be passed in via mt:entries-json element in RSS entry. The text contents of the elements is interpreted as JSON-formatted object.. + + } diff --git a/server/app-builder.js b/server/app-builder.js index bdadbea4..c2a285a6 100644 --- a/server/app-builder.js +++ b/server/app-builder.js @@ -23,6 +23,7 @@ const api = require('./routes/api'); const reports = require('./routes/reports'); const quickReports = require('./routes/quick-reports'); const subscriptions = require('./routes/subscriptions'); +const campaigns = require('./routes/campaigns'); const subscription = require('./routes/subscription'); const sandboxedMosaico = require('./routes/sandboxed-mosaico'); const sandboxedCKEditor = require('./routes/sandboxed-ckeditor'); @@ -60,7 +61,7 @@ const index = require('./routes/index'); const interoperableErrors = require('../shared/interoperable-errors'); -const { getTrustedUrl } = require('./lib/urls'); +const { getTrustedUrl, getSandboxUrl, getPublicUrl } = require('./lib/urls'); const { AppType } = require('../shared/app'); @@ -274,6 +275,8 @@ async function createApp(appType) { useWith404Fallback('/files', files); } + useWith404Fallback('/campaigns', await campaigns.getRouter(appType)); + useWith404Fallback('/mosaico', await sandboxedMosaico.getRouter(appType)); useWith404Fallback('/ckeditor', await sandboxedCKEditor.getRouter(appType)); useWith404Fallback('/grapesjs', await sandboxedGrapesJS.getRouter(appType)); @@ -357,11 +360,21 @@ async function createApp(appType) { if (err instanceof interoperableErrors.NotLoggedInError) { return res.redirect(getTrustedUrl('/login?next=' + encodeURIComponent(req.originalUrl))); } else { + let publicPath; + if (appType === AppType.TRUSTED) { + publicPath = getTrustedUrl(); + } else if (appType === AppType.SANDBOXED) { + publicPath = getSandboxUrl(); + } else if (appType === AppType.PUBLIC) { + publicPath = getPublicUrl(); + } + log.verbose('HTTP', err); res.status(err.status || 500); res.render('error', { message: err.message, - error: config.sendStacktracesToClient ? err : {} + error: config.sendStacktracesToClient ? err : {}, + publicPath }); } } diff --git a/server/lib/feedcheck.js b/server/lib/feedcheck.js index 22e7f8ed..4c4dd8e6 100644 --- a/server/lib/feedcheck.js +++ b/server/lib/feedcheck.js @@ -5,6 +5,8 @@ const log = require('./log'); const path = require('path'); const senders = require('./senders'); const bluebird = require('bluebird'); +const feedparser = require('feedparser-promised'); +const {getPublicUrl} = require('./urls'); let messageTid = 0; let feedcheckProcess; @@ -42,5 +44,79 @@ function scheduleCheck() { messageTid++; } +async function fetch(url) { + const httpOptions = { + uri: url, + headers: { + 'user-agent': 'Mailtrain', + 'accept': 'text/html,application/xhtml+xml' + } + }; + + const items = await feedparser.parse(httpOptions); + + const entries = []; + for (const item of items) { + const entry = { + title: item.title, + date: item.date || item.pubdate || item.pubDate || new Date(), + guid: item.guid || item.link, + link: item.link, + content: item.description || item.summary, + summary: item.summary || item.description, + imageUrl: item.image.url, + }; + + if ('mt:entries-json' in item) { + entry.customTags = JSON.parse(item['mt:entries-json']['#']) + } + + entries.push(entry); + } + + return entries; +} + +async function getEntryForPreview(url) { + const entries = await fetch(url); + + let entry; + if (entries.length === 0) { + entry = { + title: "Lorem Ipsum", + date: new Date(), + guid: "c21bc6c8-d351-4000-aa1f-e7ff928084cd", + link: "http://www.example.com/sample-item.html", + content: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer gravida a purus in commodo. Sed risus eros, pharetra sit amet sagittis vel, porta nec magna. Sed sollicitudin blandit ornare. Pellentesque a lacinia dui. Etiam ullamcorper, nisl at pharetra fringilla, enim nunc blandit quam, nec vestibulum purus lorem in urna.", + summary: "Aliquam malesuada nibh eget arcu egestas, id pellentesque urna egestas. Phasellus lacus est, viverra in dolor quis, aliquet elementum nisi. Donec hendrerit elit pretium vehicula pharetra. Pellentesque aliquam elit id rutrum imperdiet. Phasellus ac enim at lacus sodales condimentum vitae quis sapien.", + imageUrl: getPublicUrl('static/mailtrain-notext.png'), + customTags: { + placerat: "Ligula at consequat", + accumsan: { + mauris: "Placerat nec justo", + ornare: "Nunc egestas" + }, + fringilla: 42, + purus: [ + { + consequat: "Vivamus", + enim: "volutpat blandit" + }, + { + consequat: "Phasellus", + enim: "sed semper" + } + ] + } + }; + } else { + entry = entries[0]; + } + + return entry; +} + module.exports.spawn = bluebird.promisify(spawn); -module.exports.scheduleCheck = scheduleCheck; \ No newline at end of file +module.exports.scheduleCheck = scheduleCheck; +module.exports.fetch = fetch; +module.exports.getEntryForPreview = getEntryForPreview; \ No newline at end of file diff --git a/server/lib/message-sender.js b/server/lib/message-sender.js index 7b0360ef..fa5abfa7 100644 --- a/server/lib/message-sender.js +++ b/server/lib/message-sender.js @@ -43,7 +43,7 @@ class MessageSender { Option #1 - settings.type in [MessageType.REGULAR, MessageType.TRIGGERED, MessageType.TEST] - - campaignCid / campaignId + - campaign / campaignCid / campaignId - listId / listCid [optional if campaign is provided] - sendConfigurationId [optional if campaign is provided] - attachments [optional] @@ -68,7 +68,9 @@ class MessageSender { if (this.type === MessageType.REGULAR || this.type === MessageType.TRIGGERED || this.type === MessageType.TEST) { this.isMassMail = true; - if (settings.campaignCid) { + if (settings.campaign) { + this.campaign = settings.campaign; + } else if (settings.campaignCid) { this.campaign = await campaigns.rawGetByTx(tx, 'cid', settings.campaignCid); } else if (settings.campaignId) { this.campaign = await campaigns.rawGetByTx(tx, 'id', settings.campaignId); @@ -155,6 +157,12 @@ class MessageSender { this.tagLanguage = this.campaign.data.sourceCustom.tag_language; } + if (settings.rssEntry !== undefined) { + this.rssEntry = settings.rssEntry; + } else if (this.campaign && this.campaign.data.rssEntry) { + this.rssEntry = this.campaign.data.rssEntry; + } + enforce(this.renderedHtml || (this.campaign && this.campaign.source === CampaignSource.URL) || this.tagLanguage); if (settings.subject !== undefined) { @@ -245,11 +253,11 @@ class MessageSender { }; } - _getExtraTags(campaign) { + _getExtraTags() { const tags = {}; - if (campaign && campaign.type === CampaignType.RSS_ENTRY) { - const rssEntry = campaign.data.rssEntry; + if (this.rssEntry) { + const rssEntry = this.rssEntry; tags['RSS_ENTRY_TITLE'] = rssEntry.title; tags['RSS_ENTRY_DATE'] = rssEntry.date; tags['RSS_ENTRY_LINK'] = rssEntry.link; @@ -311,7 +319,7 @@ class MessageSender { const flds = this.listsFieldsGrouped.get(list.id); if (!mergeTags) { - mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags(campaign)); + mergeTags = fields.getMergeTags(flds, subscriptionGrouped, this._getExtraTags()); } for (const fld of flds) { @@ -539,7 +547,8 @@ async function sendQueuedMessage(queuedMessage) { subject: msgData.subject, tagLanguage: msgData.tagLanguage, renderedHtml: msgData.renderedHtml, - renderedText: msgData.renderedText + renderedText: msgData.renderedText, + rssEntry: msgData.rssEntry }); const campaign = cs.campaign; @@ -696,9 +705,9 @@ async function queueSubscriptionMessage(sendConfigurationId, to, subject, encryp senders.scheduleCheck(); } -async function getMessage(campaignCid, listCid, subscriptionCid) { +async function getMessage(campaignCid, listCid, subscriptionCid, settings) { const cs = new MessageSender(); - await cs._init({type: MessageType.REGULAR, campaignCid, listCid}); + await cs._init({type: MessageType.REGULAR, campaignCid, listCid, ...settings}); const campaign = cs.campaign; const list = cs.listsByCid.get(listCid); @@ -732,7 +741,7 @@ async function getMessage(campaignCid, listCid, subscriptionCid) { } const flds = cs.listsFieldsGrouped.get(list.id); - const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, cs._getExtraTags(campaign)); + const mergeTags = fields.getMergeTags(flds, subscriptionGrouped, cs._getExtraTags()); return await cs._getMessage(mergeTags, list, subscriptionGrouped, false); } diff --git a/server/models/campaigns.js b/server/models/campaigns.js index 7c8c5bfa..2607d42b 100644 --- a/server/models/campaigns.js +++ b/server/models/campaigns.js @@ -358,7 +358,7 @@ async function rawGetByTx(tx, key, id) { .first(); if (!entity) { - throw new interoperableErrors.NotFoundError(); + throw new shares.throwPermissionDenied(); } if (entity.lists) { @@ -425,6 +425,16 @@ async function getById(context, id, withPermissions = true, content = Content.AL }); } +async function getByCid(context, cid) { + return await knex.transaction(async tx => { + const entity = await rawGetByTx(tx,'cid', cid); + + await shares.enforceEntityPermissionTx(tx, context, 'campaign', entity.id, 'view'); + + return entity; + }); +} + async function _validateAndPreprocess(tx, context, entity, isCreate, content) { if (content === Content.ALL || content === Content.WITHOUT_SOURCE_CUSTOM || content === Content.RSS_ENTRY) { await namespaceHelpers.validateEntity(tx, entity); @@ -991,6 +1001,10 @@ async function testSend(context, data) { attachments: [] }; + if (campaign.type === CampaignType.RSS) { + messageData.rssEntry = await feedcheck.getEntryForPreview(campaign.data.feedUrl); + } + const attachments = await files.listTx(tx, contextHelpers.getAdminContext(), 'campaign', 'attachment', campaignId); for (const attachment of attachments) { messageData.attachments.push({ @@ -1057,6 +1071,30 @@ async function testSend(context, data) { senders.scheduleCheck(); } +async function getRssPreview(context, campaignCid, listCid, subscriptionCid) { + const campaign = await getByCid(context, campaignCid); + await shares.enforceEntityPermission(context, 'campaign', campaign.id, 'view'); + + enforce(campaign.type === CampaignType.RSS); + + const list = await lists.getByCid(context, listCid); + await shares.enforceEntityPermission(context, 'list', list.id, 'viewTestSubscriptions'); + + const subscription = await subscriptions.getByCid(context, list.id, subscriptionCid); + + if (!subscription.is_test) { + shares.throwPermissionDenied(); + } + + const settings = { + campaign, // this prevents message sender from fetching the campaign again + rssEntry: await feedcheck.getEntryForPreview(campaign.data.feedUrl) + }; + + return await messageSender.getMessage(campaignCid, listCid, subscriptionCid, settings); +} + + module.exports.Content = Content; module.exports.hash = hash; @@ -1073,6 +1111,7 @@ module.exports.listLinkClicksDTAjax = listLinkClicksDTAjax; module.exports.getByIdTx = getByIdTx; module.exports.getById = getById; +module.exports.getByCid = getByCid; module.exports.create = create; module.exports.createRssTx = createRssTx; module.exports.updateWithConsistencyCheck = updateWithConsistencyCheck; @@ -1101,4 +1140,5 @@ module.exports.getStatisticsOpened = getStatisticsOpened; module.exports.fetchRssCampaign = fetchRssCampaign; -module.exports.testSend = testSend; \ No newline at end of file +module.exports.testSend = testSend; +module.exports.getRssPreview = getRssPreview; \ No newline at end of file diff --git a/server/models/subscriptions.js b/server/models/subscriptions.js index 2e90be43..e4e97531 100644 --- a/server/models/subscriptions.js +++ b/server/models/subscriptions.js @@ -420,7 +420,7 @@ async function list(context, listId, grouped, offset, limit) { } async function listTestUsersTx(tx, context, listId, segmentId, grouped) { - await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewSubscriptions'); + await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewTestSubscriptions'); let entitiesQry = tx(getSubscriptionTableName(listId)).orderBy('id', 'asc').where('is_test', true).limit(TEST_USERS_LIST_LIMIT); diff --git a/server/routes/campaigns.js b/server/routes/campaigns.js new file mode 100644 index 00000000..da61caa9 --- /dev/null +++ b/server/routes/campaigns.js @@ -0,0 +1,69 @@ +'use strict'; + +const passport = require('../lib/passport'); +const routerFactory = require('../lib/router-async'); +const campaigns = require('../models/campaigns'); +const lists = require('../models/lists'); +const users = require('../models/users'); +const contextHelpers = require('../lib/context-helpers'); +const { AppType } = require('../../shared/app'); + + +users.registerRestrictedAccessTokenMethod('rssPreview', async ({campaignCid, listCid}) => { + + const campaign = await campaigns.getByCid(contextHelpers.getAdminContext(), campaignCid); + const list = await lists.getByCid(contextHelpers.getAdminContext(), listCid); + + return { + permissions: { + 'campaign': { + [campaign.id]: new Set(['view']) + }, + 'list': { + [list.id]: new Set(['view', 'viewTestSubscriptions']) + } + } + }; +}); + +async function getRouter(appType) { + const router = routerFactory.create(); + + if (appType === AppType.SANDBOXED) { + + router.get('/rss-preview/:campaign/:list/:subscription', passport.loggedIn, (req, res, next) => { + campaigns.getRssPreview(req.context, req.params.campaign, req.params.list, req.params.subscription) + .then(result => { + const {html} = result; + + if (html.match(/<\/body\b/i)) { + res.render('partials/tracking-scripts', { + layout: 'archive/layout-raw' + }, (err, scripts) => { + if (err) { + return next(err); + } + const htmlWithScripts = scripts ? html.replace(/<\/body\b/i, match => scripts + match) : html; + + res.render('archive/view', { + layout: 'archive/layout-raw', + message: htmlWithScripts + }); + }); + + } else { + res.render('archive/view', { + layout: 'archive/layout-wrapped', + message: html + }); + } + + }) + .catch(err => next(err)); + }); + } + + return router; +} + +module.exports.getRouter = getRouter; diff --git a/server/routes/subscriptions.js b/server/routes/subscriptions.js index 447348fe..b5b9306a 100644 --- a/server/routes/subscriptions.js +++ b/server/routes/subscriptions.js @@ -1,8 +1,6 @@ 'use strict'; const passport = require('../lib/passport'); -const shares = require('../models/shares'); -const contextHelpers = require('../lib/context-helpers'); const router = require('../lib/router-async').create(); const subscriptions = require('../models/subscriptions'); const {castToInteger} = require('../lib/helpers'); diff --git a/server/services/feedcheck.js b/server/services/feedcheck.js index d97284b0..b731649d 100644 --- a/server/services/feedcheck.js +++ b/server/services/feedcheck.js @@ -3,10 +3,10 @@ const config = require('../lib/config'); const log = require('../lib/log'); const knex = require('../lib/knex'); -const feedparser = require('feedparser-promised'); const { CampaignType, CampaignStatus, CampaignSource } = require('../../shared/campaigns'); const campaigns = require('../models/campaigns'); const contextHelpers = require('../lib/context-helpers'); +const {fetch} = require('../lib/feedcheck'); require('../lib/fork'); const { tLog } = require('../lib/translate'); @@ -19,39 +19,6 @@ let running = false; let periodicTimeout = null; -async function fetch(url) { - const httpOptions = { - uri: url, - headers: { - 'user-agent': 'Mailtrain', - 'accept': 'text/html,application/xhtml+xml' - } - }; - - const items = await feedparser.parse(httpOptions); - - const entries = []; - for (const item of items) { - const entry = { - title: item.title, - date: item.date || item.pubdate || item.pubDate || new Date(), - guid: item.guid || item.link, - link: item.link, - content: item.description || item.summary, - summary: item.summary || item.description, - imageUrl: item.image.url, - }; - - if ('mt:entries-json' in item) { - entry.customTags = JSON.parse(item['mt:entries-json']['#']) - } - - entries.push(entry); - } - - return entries; -} - async function run() { if (running) { return; From 23e683192f7a589fb0490e72e3d63a68e4dd1397 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Sun, 11 Aug 2019 21:01:01 +0200 Subject: [PATCH 4/9] Additional fix for #660 Fix for #662 --- client/src/campaigns/Status.js | 2 +- locales/en-US/common.json | 2 +- server/app-builder.js | 2 +- server/lib/message-sender.js | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/client/src/campaigns/Status.js b/client/src/campaigns/Status.js index 6cfd8b92..ef1c9617 100644 --- a/client/src/campaigns/Status.js +++ b/client/src/campaigns/Status.js @@ -72,7 +72,7 @@ class PreviewForTestUserModalDialog extends Component { }); const accessToken = result.data; - window.open(getSandboxUrl(`campaigns/rss-preview/${campaignCid}/${listCid}/${subscriptionCid}`, accessToken, {withLocale: true}), '_blank'); + window.open(getSandboxUrl(`cpgs/rss-preview/${campaignCid}/${listCid}/${subscriptionCid}`, accessToken, {withLocale: true}), '_blank'); } else if (entity.type === CampaignType.REGULAR) { window.open(getPublicUrl(`archive/${campaignCid}/${listCid}/${subscriptionCid}`, {withLocale: true}), '_blank'); diff --git a/locales/en-US/common.json b/locales/en-US/common.json index 6428b301..fa23af49 100644 --- a/locales/en-US/common.json +++ b/locales/en-US/common.json @@ -157,7 +157,7 @@ "campaign_plural": "Campaigns", "contentOfTheSelectedCampaignWillBeCopied": "Content of the selected campaign will be copied into this campaign.", "renderUrl": "Render URL", - "ifAMessageIsSentThenThisUrlWillBePosTed": "If a message is sent then this URL will be POSTed to using Merge Tags as POST body. Use this if you want to generate the HTML message yourself.", + "ifAMessageIsSentThenThisUrlWillBePosTed": "If a message is sent then this URL will be POSTed to using Merge Tags as POST body. Use this if you want to generate the HTML message yourself. Example: http://www.example.com/foo", "deletingCampaign": "Deleting campaign ...", "campaignDeleted": "Campaign deleted", "formCannotBeEditedBecauseTheCampaignIs": "Form cannot be edited because the campaign is currently being sent out. Wait till the sending is finished and refresh.", diff --git a/server/app-builder.js b/server/app-builder.js index c2a285a6..a6c1a984 100644 --- a/server/app-builder.js +++ b/server/app-builder.js @@ -275,7 +275,7 @@ async function createApp(appType) { useWith404Fallback('/files', files); } - useWith404Fallback('/campaigns', await campaigns.getRouter(appType)); + useWith404Fallback('/cpgs', await campaigns.getRouter(appType)); // This needs to be different from "campaigns", which is already used by the UI useWith404Fallback('/mosaico', await sandboxedMosaico.getRouter(appType)); useWith404Fallback('/ckeditor', await sandboxedCKEditor.getRouter(appType)); diff --git a/server/lib/message-sender.js b/server/lib/message-sender.js index fa5abfa7..ec26d3e0 100644 --- a/server/lib/message-sender.js +++ b/server/lib/message-sender.js @@ -200,14 +200,16 @@ class MessageSender { form[key] = mergeTags[key]; } + const sourceUrl = campaign.data.sourceUrl; + const response = await request.post({ - uri: campaign.sourceUrl, + uri: sourceUrl, form, resolveWithFullResponse: true }); if (response.statusCode !== 200) { - throw new Error(`Received status code ${httpResponse.statusCode} from ${campaign.sourceUrl}`); + throw new Error(`Received status code ${httpResponse.statusCode} from ${sourceUrl}`); } html = response.body; From bb237b3da47c2519419f7c061eb9c9bccaaaaa8d Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Sun, 11 Aug 2019 21:50:06 +0200 Subject: [PATCH 5/9] Fix - URL bases replacement didn't work for HBS tag language. --- client/src/campaigns/CUD.js | 2 +- client/src/campaigns/Content.js | 2 +- client/src/lib/sandbox-common.js | 9 +++++++ client/src/lib/sandboxed-ckeditor-root.js | 5 ++-- client/src/lib/sandboxed-ckeditor.js | 2 ++ client/src/lib/sandboxed-codeeditor-root.js | 9 ++++--- client/src/lib/sandboxed-codeeditor.js | 2 ++ client/src/lib/sandboxed-grapesjs-root.js | 11 ++++---- client/src/lib/sandboxed-grapesjs.js | 2 ++ client/src/lib/sandboxed-mosaico-root.js | 11 ++++---- client/src/lib/sandboxed-mosaico.js | 2 ++ client/src/templates/CUD.js | 2 +- client/src/templates/helpers.js | 5 ---- server/lib/campaign-content.js | 12 ++++++--- server/routes/sandboxed-mosaico.js | 2 +- shared/templates.js | 28 ++++++++++----------- 16 files changed, 63 insertions(+), 43 deletions(-) create mode 100644 client/src/lib/sandbox-common.js diff --git a/client/src/campaigns/CUD.js b/client/src/campaigns/CUD.js index 945965a9..3ce6b395 100644 --- a/client/src/campaigns/CUD.js +++ b/client/src/campaigns/CUD.js @@ -721,7 +721,7 @@ export default class CUD extends Component { templateEdit =
- + {customTemplateTypeForm}
; diff --git a/client/src/campaigns/Content.js b/client/src/campaigns/Content.js index 4880538a..3b654ce6 100644 --- a/client/src/campaigns/Content.js +++ b/client/src/campaigns/Content.js @@ -277,7 +277,7 @@ export default class CustomContent extends Component { {customTemplateTypeKey && this.templateTypes[customTemplateTypeKey].typeName} - + {customTemplateTypeKey && getTypeForm(this, customTemplateTypeKey, true)} diff --git a/client/src/lib/sandbox-common.js b/client/src/lib/sandbox-common.js new file mode 100644 index 00000000..a5b93441 --- /dev/null +++ b/client/src/lib/sandbox-common.js @@ -0,0 +1,9 @@ +'use strict'; + +export function getTagLanguageFromEntity(entity, entityTypeId) { + if (entityTypeId === 'template') { + return entity.tag_language; + } else if (entityTypeId === 'campaign') { + return entity.data.sourceCustom.tag_language; + } +} \ No newline at end of file diff --git a/client/src/lib/sandboxed-ckeditor-root.js b/client/src/lib/sandboxed-ckeditor-root.js index 6ab2f49f..d09159b9 100644 --- a/client/src/lib/sandboxed-ckeditor-root.js +++ b/client/src/lib/sandboxed-ckeditor-root.js @@ -27,7 +27,7 @@ class CKEditorSandbox extends Component { const trustedUrlBase = getTrustedUrl(); const sandboxUrlBase = getSandboxUrl(); const publicUrlBase = getPublicUrl(); - const source = this.props.initialSource && base(this.props.initialSource, trustedUrlBase, sandboxUrlBase, publicUrlBase); + const source = this.props.initialSource && base(this.props.initialSource, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase); this.state = { source @@ -37,6 +37,7 @@ class CKEditorSandbox extends Component { static propTypes = { entityTypeId: PropTypes.string, entityId: PropTypes.number, + tagLanguage: PropTypes.string, initialSource: PropTypes.string } @@ -48,7 +49,7 @@ class CKEditorSandbox extends Component { const preHtml = ''; const postHtml = ''; - const unbasedSource = unbase(this.state.source, trustedUrlBase, sandboxUrlBase, publicUrlBase, true); + const unbasedSource = unbase(this.state.source, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true); return { source: unbasedSource, diff --git a/client/src/lib/sandboxed-ckeditor.js b/client/src/lib/sandboxed-ckeditor.js index 7e0c96d1..f9b644af 100644 --- a/client/src/lib/sandboxed-ckeditor.js +++ b/client/src/lib/sandboxed-ckeditor.js @@ -11,6 +11,7 @@ import {getTrustedUrl} from "./urls"; import {initialHeight} from "./sandboxed-ckeditor-shared"; import {withComponentMixins} from "./decorator-helpers"; +import {getTagLanguageFromEntity} from "./sandbox-common"; const navbarHeight = 34; // Sync this with navbarheight in sandboxed-ckeditor.scss @@ -83,6 +84,7 @@ export class CKEditorHost extends Component { const editorData = { entityTypeId: this.props.entityTypeId, entityId: this.props.entity.id, + tagLanguage: getTagLanguageFromEntity(this.props.entity, this.props.entityTypeId), initialSource: this.props.initialSource }; diff --git a/client/src/lib/sandboxed-codeeditor-root.js b/client/src/lib/sandboxed-codeeditor-root.js index 97326e06..1501985d 100644 --- a/client/src/lib/sandboxed-codeeditor-root.js +++ b/client/src/lib/sandboxed-codeeditor-root.js @@ -66,7 +66,7 @@ class CodeEditorSandbox extends Component { const trustedUrlBase = getTrustedUrl(); const sandboxUrlBase = getSandboxUrl(); const publicUrlBase = getPublicUrl(); - const source = this.props.initialSource ? base(this.props.initialSource, trustedUrlBase, sandboxUrlBase, publicUrlBase) : defaultSource; + const source = this.props.initialSource ? base(this.props.initialSource, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase) : defaultSource; this.state = { source, @@ -87,6 +87,7 @@ class CodeEditorSandbox extends Component { static propTypes = { entityTypeId: PropTypes.string, entityId: PropTypes.number, + tagLanguage: PropTypes.string, initialSource: PropTypes.string, sourceType: PropTypes.string, initialPreview: PropTypes.bool, @@ -98,8 +99,8 @@ class CodeEditorSandbox extends Component { const sandboxUrlBase = getSandboxUrl(); const publicUrlBase = getPublicUrl(); return { - html: unbase(this.getHtml(), trustedUrlBase, sandboxUrlBase, publicUrlBase, true), - source: unbase(this.state.source, trustedUrlBase, sandboxUrlBase, publicUrlBase, true) + html: unbase(this.getHtml(), this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true), + source: unbase(this.state.source, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true) }; } @@ -149,7 +150,7 @@ class CodeEditorSandbox extends Component { }); if (!this.refreshTimeoutId) { - this.refreshTimeoutId = setTimeout(() => this.refresh(), refreshTimeout); + this.refreshTimeoutId = setTimeout(this.refreshHandler, refreshTimeout); } } diff --git a/client/src/lib/sandboxed-codeeditor.js b/client/src/lib/sandboxed-codeeditor.js index 21d59a8a..3f2f5420 100644 --- a/client/src/lib/sandboxed-codeeditor.js +++ b/client/src/lib/sandboxed-codeeditor.js @@ -9,6 +9,7 @@ import {UntrustedContentHost} from './untrusted'; import {Icon} from "./bootstrap-components"; import {getTrustedUrl} from "./urls"; import {withComponentMixins} from "./decorator-helpers"; +import {getTagLanguageFromEntity} from "./sandbox-common"; @withComponentMixins([ withTranslation @@ -75,6 +76,7 @@ export class CodeEditorHost extends Component { const editorData = { entityTypeId: this.props.entityTypeId, entityId: this.props.entity.id, + tagLanguage: getTagLanguageFromEntity(this.props.entity, this.props.entityTypeId), initialSource: this.props.initialSource, sourceType: this.props.sourceType, initialPreview: this.state.preview, diff --git a/client/src/lib/sandboxed-grapesjs-root.js b/client/src/lib/sandboxed-grapesjs-root.js index f9c2f76d..00d89fe6 100644 --- a/client/src/lib/sandboxed-grapesjs-root.js +++ b/client/src/lib/sandboxed-grapesjs-root.js @@ -54,6 +54,7 @@ export class GrapesJSSandbox extends Component { static propTypes = { entityTypeId: PropTypes.string, entityId: PropTypes.number, + tagLanguage: PropTypes.string, initialSource: PropTypes.string, initialStyle: PropTypes.string, sourceType: PropTypes.string @@ -75,8 +76,8 @@ export class GrapesJSSandbox extends Component { const sandboxUrlBase = getSandboxUrl(); const publicUrlBase = getPublicUrl(); - const source = unbase(editor.getHtml(), trustedUrlBase, sandboxUrlBase, publicUrlBase, true); - const style = unbase(editor.getCss(), trustedUrlBase, sandboxUrlBase, publicUrlBase, true); + const source = unbase(editor.getHtml(), this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true); + const style = unbase(editor.getCss(), this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true); let html; @@ -96,7 +97,7 @@ export class GrapesJSSandbox extends Component { const preHtml = ''; const postHtml = ''; - html = preHtml + unbase(htmlBody, trustedUrlBase, sandboxUrlBase, publicUrlBase, true) + postHtml; + html = preHtml + unbase(htmlBody, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true) + postHtml; } @@ -603,8 +604,8 @@ export class GrapesJSSandbox extends Component { config.plugins.push('gjs-preset-newsletter'); } - config.components = props.initialSource ? base(props.initialSource, trustedUrlBase, sandboxUrlBase, publicUrlBase) : defaultSource; - config.style = props.initialStyle ? base(props.initialStyle, trustedUrlBase, sandboxUrlBase, publicUrlBase) : defaultStyle; + config.components = props.initialSource ? base(props.initialSource, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase) : defaultSource; + config.style = props.initialStyle ? base(props.initialStyle, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase) : defaultStyle; config.plugins.push('mailtrain-remove-buttons'); diff --git a/client/src/lib/sandboxed-grapesjs.js b/client/src/lib/sandboxed-grapesjs.js index 42fa23a4..d3de59e3 100644 --- a/client/src/lib/sandboxed-grapesjs.js +++ b/client/src/lib/sandboxed-grapesjs.js @@ -10,6 +10,7 @@ import {Icon} from "./bootstrap-components"; import {getTrustedUrl} from "./urls"; import {withComponentMixins} from "./decorator-helpers"; import {GrapesJSSourceType} from "./sandboxed-grapesjs-shared"; +import {getTagLanguageFromEntity} from "./sandbox-common"; @withComponentMixins([ withTranslation @@ -57,6 +58,7 @@ export class GrapesJSHost extends Component { const editorData = { entityTypeId: this.props.entityTypeId, entityId: this.props.entity.id, + tagLanguage: getTagLanguageFromEntity(this.props.entity, this.props.entityTypeId), initialSource: this.props.initialSource, initialStyle: this.props.initialStyle, sourceType: this.props.sourceType diff --git a/client/src/lib/sandboxed-mosaico-root.js b/client/src/lib/sandboxed-mosaico-root.js index 4a3dfd5a..78bddac0 100644 --- a/client/src/lib/sandboxed-mosaico-root.js +++ b/client/src/lib/sandboxed-mosaico-root.js @@ -27,6 +27,7 @@ class MosaicoSandbox extends Component { static propTypes = { entityTypeId: PropTypes.string, entityId: PropTypes.number, + tagLanguage: PropTypes.string, templateId: PropTypes.number, templatePath: PropTypes.string, initialModel: PropTypes.string, @@ -60,9 +61,9 @@ class MosaicoSandbox extends Component { html = juice(html); return { - html: unbase(html, trustedUrlBase, sandboxUrlBase, publicUrlBase, true), - model: unbase(this.viewModel.exportJSON(), trustedUrlBase, sandboxUrlBase, publicUrlBase), - metadata: unbase(this.viewModel.exportMetadata(), trustedUrlBase, sandboxUrlBase, publicUrlBase) + html: unbase(html, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase, true), + model: unbase(this.viewModel.exportJSON(), this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase), + metadata: unbase(this.viewModel.exportMetadata(), this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase) }; } @@ -128,8 +129,8 @@ class MosaicoSandbox extends Component { const trustedUrlBase = getTrustedUrl(); const sandboxUrlBase = getSandboxUrl(); const publicUrlBase = getPublicUrl(); - const metadata = this.props.initialMetadata && JSON.parse(base(this.props.initialMetadata, trustedUrlBase, sandboxUrlBase, publicUrlBase)); - const model = this.props.initialModel && JSON.parse(base(this.props.initialModel, trustedUrlBase, sandboxUrlBase, publicUrlBase)); + const metadata = this.props.initialMetadata && JSON.parse(base(this.props.initialMetadata, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase)); + const model = this.props.initialModel && JSON.parse(base(this.props.initialModel, this.props.tagLanguage, trustedUrlBase, sandboxUrlBase, publicUrlBase)); const template = this.props.templateId ? getSandboxUrl(`mosaico/templates/${this.props.templateId}/index.html`) : this.props.templatePath; const allPlugins = plugins.concat(window.mosaicoPlugins); diff --git a/client/src/lib/sandboxed-mosaico.js b/client/src/lib/sandboxed-mosaico.js index 7704c0d1..352da1fd 100644 --- a/client/src/lib/sandboxed-mosaico.js +++ b/client/src/lib/sandboxed-mosaico.js @@ -9,6 +9,7 @@ import {UntrustedContentHost} from './untrusted'; import {Icon} from "./bootstrap-components"; import {getTrustedUrl} from "./urls"; import {withComponentMixins} from "./decorator-helpers"; +import {getTagLanguageFromEntity} from "./sandbox-common"; @withComponentMixins([ @@ -58,6 +59,7 @@ export class MosaicoHost extends Component { const editorData = { entityTypeId: this.props.entityTypeId, entityId: this.props.entity.id, + tagLanguage: getTagLanguageFromEntity(this.props.entity, this.props.entityTypeId), templateId: this.props.templateId, templatePath: this.props.templatePath, initialModel: this.props.initialModel, diff --git a/client/src/templates/CUD.js b/client/src/templates/CUD.js index c89166de..59dcb1f9 100644 --- a/client/src/templates/CUD.js +++ b/client/src/templates/CUD.js @@ -376,7 +376,7 @@ export default class CUD extends Component { {typeForm} - + } diff --git a/client/src/templates/helpers.js b/client/src/templates/helpers.js index dd651149..c49a5ad0 100644 --- a/client/src/templates/helpers.js +++ b/client/src/templates/helpers.js @@ -167,7 +167,6 @@ export function getTemplateTypes(t, prefix = '', entityTypeId = ResourceType.TEM mutState.setIn([prefix + 'mosaicoTemplate', 'value'], null); } }, - isTagLanguageSelectorDisabledForEdit: true, validate: state => { const mosaicoTemplate = state.getIn([prefix + 'mosaicoTemplate', 'value']); if (!mosaicoTemplate) { @@ -256,7 +255,6 @@ export function getTemplateTypes(t, prefix = '', entityTypeId = ResourceType.TEM }, afterTagLanguageChange: (mutState, isEdit) => { }, - isTagLanguageSelectorDisabledForEdit: false, validate: state => { } }; @@ -346,7 +344,6 @@ export function getTemplateTypes(t, prefix = '', entityTypeId = ResourceType.TEM }, afterTagLanguageChange: (mutState, isEdit) => { }, - isTagLanguageSelectorDisabledForEdit: false, validate: state => { } }; @@ -408,7 +405,6 @@ export function getTemplateTypes(t, prefix = '', entityTypeId = ResourceType.TEM }, afterTagLanguageChange: (mutState, isEdit) => { }, - isTagLanguageSelectorDisabledForEdit: false, validate: state => { } }; @@ -495,7 +491,6 @@ export function getTemplateTypes(t, prefix = '', entityTypeId = ResourceType.TEM }, afterTagLanguageChange: (mutState, isEdit) => { }, - isTagLanguageSelectorDisabledForEdit: false, validate: state => { } }; diff --git a/server/lib/campaign-content.js b/server/lib/campaign-content.js index 965b6bfb..19728094 100644 --- a/server/lib/campaign-content.js +++ b/server/lib/campaign-content.js @@ -1,7 +1,11 @@ 'use strict'; +const {renderTag} = require('../../shared/templates'); + function convertFileURLs(sourceCustom, fromEntityType, fromEntityId, toEntityType, toEntityId) { + const tagLanguage = sourceCustom.tag_language; + function convertText(text) { if (text) { const fromUrl = `/files/${fromEntityType}/file/${fromEntityId}`; @@ -10,10 +14,10 @@ function convertFileURLs(sourceCustom, fromEntityType, fromEntityId, toEntityTyp const encodedFromUrl = encodeURIComponent(fromUrl); const encodedToUrl = encodeURIComponent(toUrl); - text = text.split('[URL_BASE]' + fromUrl).join('[URL_BASE]' + toUrl); - text = text.split('[SANDBOX_URL_BASE]' + fromUrl).join('[SANDBOX_URL_BASE]' + toUrl); - text = text.split('[ENCODED_URL_BASE]' + encodedFromUrl).join('[ENCODED_URL_BASE]' + encodedToUrl); - text = text.split('[ENCODED_SANDBOX_URL_BASE]' + encodedFromUrl).join('[ENCODED_SANDBOX_URL_BASE]' + encodedToUrl); + text = text.split(renderTag(tagLanguage, 'URL_BASE') + fromUrl).join(renderTag(tagLanguage, 'URL_BASE') + toUrl); + text = text.split(renderTag(tagLanguage,'SANDBOX_URL_BASE') + fromUrl).join(renderTag(tagLanguage, 'SANDBOX_URL_BASE') + toUrl); + text = text.split(renderTag(tagLanguage, 'ENCODED_URL_BASE') + encodedFromUrl).join(renderTag(tagLanguage, 'ENCODED_URL_BASE') + encodedToUrl); + text = text.split(renderTag(tagLanguage, 'ENCODED_SANDBOX_URL_BASE') + encodedFromUrl).join(renderTag(tagLanguage, 'ENCODED_SANDBOX_URL_BASE') + encodedToUrl); } return text; diff --git a/server/routes/sandboxed-mosaico.js b/server/routes/sandboxed-mosaico.js index 98f4854b..ceac02b7 100644 --- a/server/routes/sandboxed-mosaico.js +++ b/server/routes/sandboxed-mosaico.js @@ -141,7 +141,7 @@ async function getRouter(appType) { const tmpl = await mosaicoTemplates.getById(req.context, castToInteger(req.params.mosaicoTemplateId)); res.set('Content-Type', 'text/html'); - res.send(base(tmpl.data.html, getTrustedUrl(), getSandboxUrl('', req.context), getPublicUrl())); + res.send(base(tmpl.data.html, tmpl.tag_language, getTrustedUrl(), getSandboxUrl('', req.context), getPublicUrl())); }); // Mosaico looks for block thumbnails in edres folder relative to index.html of the template. We respond to such requests here. diff --git a/shared/templates.js b/shared/templates.js index 35d819cf..a4c22fa8 100644 --- a/shared/templates.js +++ b/shared/templates.js @@ -44,28 +44,28 @@ function getMergeTagsForBases(trustedBaseUrl, sandboxBaseUrl, publicBaseUrl) { }; } -function base(text, trustedBaseUrl, sandboxBaseUrl, publicBaseUrl) { +function base(text, tagLanguage, trustedBaseUrl, sandboxBaseUrl, publicBaseUrl) { const bases = _getBases(trustedBaseUrl, sandboxBaseUrl, publicBaseUrl); - text = text.split('[URL_BASE]').join(bases.publicBaseUrl); - text = text.split('[TRUSTED_URL_BASE]').join(bases.trustedBaseUrl); - text = text.split('[SANDBOX_URL_BASE]').join(bases.sandboxBaseUrl); - text = text.split('[ENCODED_URL_BASE]').join(encodeURIComponent(bases.publicBaseUrl)); - text = text.split('[ENCODED_TRUSTED_URL_BASE]').join(encodeURIComponent(bases.trustedBaseUrl)); - text = text.split('[ENCODED_SANDBOX_URL_BASE]').join(encodeURIComponent(bases.sandboxBaseUrl)); + text = text.split(renderTag(tagLanguage, 'URL_BASE')).join(bases.publicBaseUrl); + text = text.split(renderTag(tagLanguage, 'TRUSTED_URL_BASE')).join(bases.trustedBaseUrl); + text = text.split(renderTag(tagLanguage, 'SANDBOX_URL_BASE')).join(bases.sandboxBaseUrl); + text = text.split(renderTag(tagLanguage, 'ENCODED_URL_BASE')).join(encodeURIComponent(bases.publicBaseUrl)); + text = text.split(renderTag(tagLanguage, 'ENCODED_TRUSTED_URL_BASE')).join(encodeURIComponent(bases.trustedBaseUrl)); + text = text.split(renderTag(tagLanguage, 'ENCODED_SANDBOX_URL_BASE')).join(encodeURIComponent(bases.sandboxBaseUrl)); return text; } -function unbase(text, trustedBaseUrl, sandboxBaseUrl, publicBaseUrl, treatAllAsPublic = false) { +function unbase(text, tagLanguage, trustedBaseUrl, sandboxBaseUrl, publicBaseUrl, treatAllAsPublic = false) { const bases = _getBases(trustedBaseUrl, sandboxBaseUrl, publicBaseUrl); - text = text.split(bases.publicBaseUrl).join('[URL_BASE]'); - text = text.split(bases.trustedBaseUrl).join(treatAllAsPublic ? '[URL_BASE]' : '[TRUSTED_URL_BASE]'); - text = text.split(bases.sandboxBaseUrl).join(treatAllAsPublic ? '[URL_BASE]' : '[SANDBOX_URL_BASE]'); - text = text.split(encodeURIComponent(bases.publicBaseUrl)).join('[ENCODED_URL_BASE]'); - text = text.split(encodeURIComponent(bases.trustedBaseUrl)).join(treatAllAsPublic ? '[ENCODED_URL_BASE]' : '[ENCODED_TRUSTED_URL_BASE]'); - text = text.split(encodeURIComponent(bases.sandboxBaseUrl)).join(treatAllAsPublic ? '[ENCODED_URL_BASE]' : '[ENCODED_SANDBOX_URL_BASE]'); + text = text.split(bases.publicBaseUrl).join(renderTag(tagLanguage, 'URL_BASE')); + text = text.split(bases.trustedBaseUrl).join(renderTag(tagLanguage, treatAllAsPublic ? 'URL_BASE' : 'TRUSTED_URL_BASE')); + text = text.split(bases.sandboxBaseUrl).join(renderTag(tagLanguage, treatAllAsPublic ? 'URL_BASE' : 'SANDBOX_URL_BASE')); + text = text.split(encodeURIComponent(bases.publicBaseUrl)).join(renderTag(tagLanguage, 'ENCODED_URL_BASE')); + text = text.split(encodeURIComponent(bases.trustedBaseUrl)).join(renderTag(tagLanguage, treatAllAsPublic ? 'ENCODED_URL_BASE' : 'ENCODED_TRUSTED_URL_BASE')); + text = text.split(encodeURIComponent(bases.sandboxBaseUrl)).join(renderTag(tagLanguage, treatAllAsPublic ? 'ENCODED_URL_BASE' : 'ENCODED_SANDBOX_URL_BASE')); return text; } From ae5faadffad95f315723a7649883a632c4a620f7 Mon Sep 17 00:00:00 2001 From: Tomas Bures Date: Mon, 12 Aug 2019 09:26:49 +0200 Subject: [PATCH 6/9] Fix for #665 and additional fix for #663. If your segemnts are broken or Mailtrain complains about missing 20190726150000_shorten_field_column_names.js, run the following in `server/setup/knex/fixes`: ```NODE_ENV=production node fix-20190726150000_shorten_field_column_names.js``` --- server/models/fields.js | 3 +- server/setup/knex/config.js | 2 +- ...190726150000_shorten_field_column_names.js | 69 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 server/setup/knex/fixes/fix-20190726150000_shorten_field_column_names.js diff --git a/server/models/fields.js b/server/models/fields.js index faac8bd7..2f7148eb 100644 --- a/server/models/fields.js +++ b/server/models/fields.js @@ -8,6 +8,7 @@ const interoperableErrors = require('../../shared/interoperable-errors'); const shares = require('./shares'); const validators = require('../../shared/validators'); const shortid = require('shortid'); +const slugify = require('slugify'); const segments = require('./segments'); const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../../shared/date'); const { getFieldColumn } = require('../../shared/lists'); @@ -542,7 +543,7 @@ async function createTx(tx, context, listId, entity) { let columnName; if (!fieldType.grouped) { - columnName = ('custom_' + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '_'); + columnName = ('custom_' + slugify(entity.name, '_').substring(0, 32) + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '_'); } const filteredEntity = filterObject(entity, allowedKeysCreate); diff --git a/server/setup/knex/config.js b/server/setup/knex/config.js index 0f86dc97..2e5cea25 100644 --- a/server/setup/knex/config.js +++ b/server/setup/knex/config.js @@ -4,6 +4,6 @@ if (!process.env.NODE_CONFIG_DIR) { process.env.NODE_CONFIG_DIR = __dirname + '/../../config'; } -const config = require('server/setup/knex/config'); +const config = require('../../lib/config'); module.exports = config; diff --git a/server/setup/knex/fixes/fix-20190726150000_shorten_field_column_names.js b/server/setup/knex/fixes/fix-20190726150000_shorten_field_column_names.js new file mode 100644 index 00000000..d83a1937 --- /dev/null +++ b/server/setup/knex/fixes/fix-20190726150000_shorten_field_column_names.js @@ -0,0 +1,69 @@ +'use strict'; + +const config = require('../config'); +const knex = require('../../../lib/knex'); +const shortid = require('shortid'); +const slugify = require('slugify'); + +async function run() { + const lists = await knex('lists'); + for (const list of lists) { + console.log(`Processing list ${list.id}`); + const fields = await knex('custom_fields').whereNotNull('column').where('list', list.id); + + const fieldsMap = new Map(); + const prefixesMap = new Map(); + + for (const field of fields) { + const oldName = field.column; + const newName = ('custom_' + slugify(field.name, '_').substring(0, 32) + '_' + shortid.generate()).toLowerCase().replace(/[^a-z0-9_]/g, '_'); + const formerPrefix = ('custom_' + slugify(field.name, '_') + '_').toLowerCase().replace(/[^a-z0-9_]/g, ''); + + fieldsMap.set(oldName, newName); + prefixesMap.set(formerPrefix, newName); + + await knex('custom_fields').where('id', field.id).update('column', newName); + + await knex.schema.table('subscription__' + list.id, table => { + table.renameColumn(oldName, newName); + table.renameColumn('source_' + oldName, 'source_' + newName); + }); + } + + + function processRule(rule) { + if (rule.type === 'all' || rule.type === 'some' || rule.type === 'none') { + for (const childRule of rule.rules) { + processRule(childRule); + } + } else { + let newName = fieldsMap.get(rule.column); + if (newName) { + rule.column = newName; + return; + } + + for (const [formerPrefix, newName] of prefixesMap.entries()) { + if (rule.column.startsWith(formerPrefix)) { + rule.column = newName; + return; + } + } + } + } + + const segments = await knex('segments').where('list', list.id); + for (const segment of segments) { + const settings = JSON.parse(segment.settings); + processRule(settings.rootRule); + await knex('segments').where('id', segment.id).update({settings: JSON.stringify(settings)}); + } + } + + await knex('knex_migrations').where('name', '20190726150000_shorten_field_column_names.js').del(); + + console.log('All fixes done'); + process.exit(); +} + +run().catch(err => console.error(err)); \ No newline at end of file From c8eeeaa9b93ab968e451eace887e7cdc40dc30d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillaume=20R=C3=A9my?= Date: Wed, 14 Aug 2019 11:10:14 +0200 Subject: [PATCH 7/9] Added secure config parameter to use ldaps protocol --- server/lib/passport.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/lib/passport.js b/server/lib/passport.js index 8b1eaae5..52423d57 100644 --- a/server/lib/passport.js +++ b/server/lib/passport.js @@ -20,7 +20,8 @@ let authMode = 'local'; let LdapStrategy; let ldapStrategyOpts; if (config.ldap.enabled) { - if (!config.ldap.method || config.ldap.method == 'ldapjs') { + const ldapProtocol = config.ldap.secure ? 'ldaps' : 'ldap'; + if (!config.ldap.method || config.ldap.method === 'ldapjs') { try { LdapStrategy = require('passport-ldapjs').Strategy; // eslint-disable-line global-require authMode = 'ldapjs'; @@ -28,7 +29,7 @@ if (config.ldap.enabled) { ldapStrategyOpts = { server: { - url: 'ldap://' + config.ldap.host + ':' + config.ldap.port + url: ldapProtocol + '://' + config.ldap.host + ':' + config.ldap.port }, base: config.ldap.baseDN, search: { @@ -46,7 +47,7 @@ if (config.ldap.enabled) { } } - if (!LdapStrategy && (!config.ldap.method || config.ldap.method == 'ldapauth')) { + if (!LdapStrategy && (!config.ldap.method || config.ldap.method === 'ldapauth')) { try { LdapStrategy = require('passport-ldapauth').Strategy; // eslint-disable-line global-require authMode = 'ldapauth'; @@ -54,7 +55,7 @@ if (config.ldap.enabled) { ldapStrategyOpts = { server: { - url: 'ldap://' + config.ldap.host + ':' + config.ldap.port, + url: ldapProtocol + '://' + config.ldap.host + ':' + config.ldap.port, searchBase: config.ldap.baseDN, searchFilter: config.ldap.filter, searchAttributes: [config.ldap.uidTag, config.ldap.nameTag, 'mail'], From 98cd14f8be1d8a4f3637595b2d88a034ff846b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillaume=20R=C3=A9my?= Date: Wed, 14 Aug 2019 11:30:03 +0200 Subject: [PATCH 8/9] Added ldap secure parameter in default.yaml config file --- server/config/default.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/config/default.yaml b/server/config/default.yaml index 39ffb95e..247d969e 100644 --- a/server/config/default.yaml +++ b/server/config/default.yaml @@ -139,6 +139,8 @@ ldap: # method: ldapjs host: localhost port: 3002 + # secure enables ldaps protocol if true. Otherwise, ldap protocol is used. + secure: false baseDN: ou=users,dc=company filter: (|(username={{username}})(mail={{username}})) # Username field in LDAP (uid/cn/username) From de20d8a64cf9dccabeb1b84c0fa765e8eb0f9968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillaume=20R=C3=A9my?= Date: Wed, 14 Aug 2019 13:32:20 +0200 Subject: [PATCH 9/9] Added new parameters for LDAP in docker-entrypoint.sh --- docker-entrypoint.sh | 90 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 6914d668..52c6ff1e 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -12,6 +12,15 @@ Optional parameters: --mongoHost XXX - sets mongo host (default: mongo) --redisHost XXX - sets redis host (default: redis) --mySqlHost XXX - sets mysql host (default: mysql) + --withLdap - use if you want to enable LDAP authentication + --ldapHost XXX - LDAP Host for authentication (default: ldap) + --ldapPort XXX - LDAP port (default: 389) + --ldapSecure - use if you want to use LDAP with ldaps protocol + --ldapBindUser XXX - User for LDAP connexion + --ldapBindPass XXX - Password for LDAP connexion + --ldapFilter XXX - LDAP filter + --ldapBaseDN XXX - LDAP base DN + --ldapUidTag XXX - LDAP UID tag (e.g. uid/cn/username) EOF exit 1 @@ -22,6 +31,15 @@ urlBaseTrusted=http://localhost:3000 urlBaseSandbox=http://localhost:3003 urlBasePublic=http://localhost:3004 wwwProxy=false +withLdap=false +ldapHost=ldap +ldapPort=389 +ldapSecure=false +ldapBindUser="" +ldapBindPass="" +ldapFilter="" +ldapBaseDN="" +ldapUidTag="" mongoHost=mongo redisHost=redis mySqlHost=mysql @@ -59,12 +77,73 @@ while [ $# -gt 0 ]; do mySqlHost="$2" shift 2 ;; + --withLdap) + withLdap=true + shift 1 + ;; + --ldapHost) + ldapHost="$2" + shift 2 + ;; + --ldapPort) + ldapPort="$2" + shift 2 + ;; + --ldapSecure) + ldapSecure=true + shift 1 + ;; + --ldapBindUser) + ldapBindUser="$2" + shift 2 + ;; + --ldapBindPass) + ldapBindPass="$2" + shift 2 + ;; + --ldapFilter) + ldapFilter="$2" + shift 2 + ;; + --ldapBaseDN) + ldapBaseDN="$2" + shift 2 + ;; + --ldapUidTag) + ldapUidTag="$2" + shift 2 + ;; *) echo "Error: unrecognized option $1." printHelp esac done +if [ "$ldapBindUser" == "" ]; then + ldapBindUserLine="" +else + ldapBindUserLine="bindUser: $ldapBindUser" +fi +if [ "$ldapBindPass" == "" ]; then + ldapBindPassLine="" +else + ldapBindPassLine="bindPassword: $ldapBindPass" +fi +if [ "$ldapFilter" == "" ]; then + ldapFilterLine="" +else + ldapFilterLine="filter: $ldapFilter" +fi +if [ "$ldapBaseDN" == "" ]; then + ldapBaseDNLine="" +else + ldapBaseDNLine="baseDN: $ldapBaseDN" +fi +if [ "$ldapUidTag" == "" ]; then + ldapUidTagLine="" +else + ldapUidTagLine="uidTag: $ldapUidTag" +fi cat > server/config/production.yaml < server/services/workers/reports/config/production.yaml <