diff --git a/client/package-lock.json b/client/package-lock.json
index 9dd6aede..fee5c792 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -7018,9 +7018,9 @@
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
},
"moment-timezone": {
- "version": "0.5.23",
- "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.23.tgz",
- "integrity": "sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==",
+ "version": "0.5.25",
+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.25.tgz",
+ "integrity": "sha512-DgEaTyN/z0HFaVcVbSyVCUU6HeFdnNC3vE4c9cgu2dgMTvjBUBdBzWfasTBmAW45u5OIMeCJtU8yNjM22DHucw==",
"requires": {
"moment": ">= 2.9.0"
}
diff --git a/client/package.json b/client/package.json
index c1b2a734..0cf573ed 100644
--- a/client/package.json
+++ b/client/package.json
@@ -38,7 +38,7 @@
"lodash": "^4.17.11",
"mjml4-in-browser": "^1.1.1",
"moment": "^2.23.0",
- "moment-timezone": "^0.5.23",
+ "moment-timezone": "^0.5.25",
"popper.js": "^1.14.6",
"prop-types": "^15.6.2",
"querystringify": "^2.1.0",
@@ -56,9 +56,9 @@
"react-i18next": "^9.0.1",
"react-router-dom": "^4.3.1",
"react-sortable-tree": "^2.6.0",
- "slugify": "^1.3.4",
"shallowequal": "^1.1.0",
"shortid": "^2.2.14",
+ "slugify": "^1.3.4",
"url-parse": "^1.4.4"
},
"devDependencies": {
diff --git a/client/src/reports/ViewAndOutput.js b/client/src/reports/ViewAndOutput.js
index 979db746..557bb019 100644
--- a/client/src/reports/ViewAndOutput.js
+++ b/client/src/reports/ViewAndOutput.js
@@ -45,12 +45,14 @@ export default class ViewAndOutput extends Component {
url: 'rest/report-content',
getTitle: name => t('reportName', { name }),
loading: t('loadingReport'),
+ finishedStates: new Set([ReportState.FINISHED]),
getContent: content =>
},
output: {
url: 'rest/report-output',
getTitle: name => t('outputForReportName', { name }),
loading: t('loadingReportOutput'),
+ finishedStates: new Set([ReportState.FINISHED, ReportState.FAILED]),
getContent: content => {content}
}
}
@@ -108,7 +110,7 @@ export default class ViewAndOutput extends Component {
if (this.state.report) {
let reportContent = null;
- if (this.state.report.state === ReportState.FINISHED) {
+ if (viewType.finishedStates.has(this.state.report.state)) {
reportContent = viewType.getContent(this.state.content);
} else if (this.state.report.state === ReportState.SCHEDULED || this.state.report.state === ReportState.PROCESSING) {
reportContent = {t('reportIsBeingGenerated')}
;
diff --git a/client/src/reports/templates/CUD.js b/client/src/reports/templates/CUD.js
index 45a8b6ab..2c5f80bd 100644
--- a/client/src/reports/templates/CUD.js
+++ b/client/src/reports/templates/CUD.js
@@ -130,23 +130,17 @@ export default class CUD extends Component {
' "maxOccurences": 1\n' +
' }\n' +
']',
- js:
- 'const sampleRowTransform = new stream.Transform({\n' +
- ' objectMode: true,\n' +
- ' transform(row, encoding, callback) {\n' +
- ' callback(null, row)\n' +
- ' }\n' +
- '})\n' +
+ js: 'const results = await campaigns.getCampaignOpenStatisticsStream(inputs.campaign, [\'subscription:email\', \'tracker:count\'], null, (query, col) => query.where(col(\'subscription:status\'), SubscriptionStatus.SUBSCRIBED));\n' +
'\n' +
- 'const results = await campaigns.getCampaignOpenStatisticsStream(inputs.campaign, [\'subscription:email\', \'tracker:count\'])\n' +
- '\n' +
- 'results.pipe(sampleRowTransform)\n' +
- '\n' +
- 'await renderCsvFromStream(sampleRowTransform, {\n' +
- ' header: true,\n' +
- ' columns: [ { key: \'subscription:email\', header: \'Email\' }, { key: \'tracker:count\', header: \'Open count\' } ],\n' +
- ' delimiter: \',\'\n' +
- '})',
+ 'await renderCsvFromStream(\n' +
+ ' results, \n' +
+ ' {\n' +
+ ' header: true,\n' +
+ ' columns: [ { key: \'subscription:email\', header: \'Email\' }, { key: \'tracker:count\', header: \'Open count\' } ],\n' +
+ ' delimiter: \',\'\n' +
+ ' },\n' +
+ ' async (row, encoding) => row\n' +
+ ');',
hbs: ''
});
@@ -167,9 +161,9 @@ export default class CUD extends Component {
' }\n' +
']',
js:
- 'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["field:country", "count_opened", "count_all"], query =>\n' +
+ 'const results = await campaigns.getCampaignOpenStatistics(inputs.campaign, ["field:country", "count_opened", "count_all"], (query, col) =>\n' +
' query.count("* AS count_all")\n' +
- ' .select(knex.raw("SUM(IF(`tracker:count` IS NULL, 0, 1)) AS count_opened"))\n' +
+ ' .select(knex.raw("SUM(IF(`" + col(tracker:count) +"` IS NULL, 0, 1)) AS count_opened"))\n' +
' .groupBy("field:country")\n' +
')\n' +
'\n' +
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index ce2ba6e8..f8d9f0d6 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -30,12 +30,10 @@ services:
- "3004:3004"
volumes:
- mailtrain-files:/app/server/files
- - mailtrain-reports:/app/protected/reports
volumes:
mysql-data:
redis-data:
mongo-data:
mailtrain-files:
- mailtrain-reports:
diff --git a/docker-compose.yml b/docker-compose.yml
index 61055326..4bf82f74 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -30,12 +30,10 @@ services:
- "3004:3004"
volumes:
- mailtrain-files:/app/server/files
- - mailtrain-reports:/app/protected/reports
volumes:
mysql-data:
redis-data:
mongo-data:
mailtrain-files:
- mailtrain-reports:
diff --git a/server/app-builder.js b/server/app-builder.js
index 55c84c50..3203643b 100644
--- a/server/app-builder.js
+++ b/server/app-builder.js
@@ -118,7 +118,7 @@ hbs.registerHelper('flash_messages', function () { // eslint-disable-line prefer
-function createApp(appType) {
+async function createApp(appType) {
const app = express();
function install404Fallback(url) {
@@ -273,10 +273,10 @@ function createApp(appType) {
useWith404Fallback('/files', files);
}
- useWith404Fallback('/mosaico', sandboxedMosaico.getRouter(appType));
- useWith404Fallback('/ckeditor', sandboxedCKEditor.getRouter(appType));
- useWith404Fallback('/grapesjs', sandboxedGrapesJS.getRouter(appType));
- useWith404Fallback('/codeeditor', sandboxedCodeEditor.getRouter(appType));
+ useWith404Fallback('/mosaico', await sandboxedMosaico.getRouter(appType));
+ useWith404Fallback('/ckeditor', await sandboxedCKEditor.getRouter(appType));
+ useWith404Fallback('/grapesjs', await sandboxedGrapesJS.getRouter(appType));
+ useWith404Fallback('/codeeditor', await sandboxedCodeEditor.getRouter(appType));
if (appType === AppType.TRUSTED || appType === AppType.SANDBOXED) {
useWith404Fallback('/subscriptions', subscriptions);
@@ -318,7 +318,7 @@ function createApp(appType) {
install404Fallback('/rest');
}
- app.use('/', index.getRouter(appType));
+ app.use('/', await index.getRouter(appType));
app.use((err, req, res, next) => {
if (!err) {
diff --git a/server/config/default.yaml b/server/config/default.yaml
index 9bc88cf3..b25c0cdb 100644
--- a/server/config/default.yaml
+++ b/server/config/default.yaml
@@ -176,6 +176,12 @@ mosaico:
# Inject custom scripts
# customscripts:
# - /mosaico/custom/my-mosaico-plugin.js
+ fileCache:
+ blockThumbnails:
+ maxSize: 100 # megabytes
+ images:
+ maxSize: 1024 # megabytes
+
grapesjs:
# Installed templates
diff --git a/server/index.js b/server/index.js
index 09167f24..efce879d 100644
--- a/server/index.js
+++ b/server/index.js
@@ -19,11 +19,13 @@ const reportProcessor = require('./lib/report-processor');
const executor = require('./lib/executor');
const privilegeHelpers = require('./lib/privilege-helpers');
const knex = require('./lib/knex');
+const bluebird = require('bluebird');
const shares = require('./models/shares');
const { AppType } = require('../shared/app');
const builtinZoneMta = require('./lib/builtin-zone-mta');
const { uploadedFilesDir } = require('./lib/file-helpers');
+const { reportFilesDir } = require('./lib/report-helpers');
const { filesDir } = require('./models/files');
const trustedPort = config.www.trustedPort;
@@ -36,8 +38,8 @@ if (config.title) {
}
-function startHTTPServer(appType, appName, port, callback) {
- const app = appBuilder.createApp(appType);
+async function startHTTPServer(appType, appName, port) {
+ const app = await appBuilder.createApp(appType);
app.set('port', port);
const server = http.createServer(app);
@@ -68,81 +70,58 @@ function startHTTPServer(appType, appName, port, callback) {
log.info('Express', 'WWW server [%s] listening on %s', appName, bind);
});
- server.listen({port, host}, callback);
+ const serverListenAsync = bluebird.promisify(server.listen.bind(server));
+ await serverListenAsync({port, host});
}
+
// ---------------------------------------------------------------------------------------
-// Start the whole circus here
+// Start the whole circus
// ---------------------------------------------------------------------------------------
-dbcheck(err => { // Check if database needs upgrading before starting the server - legacy migration first
- if (err) {
- log.error('DB', err.message || err);
- return process.exit(1);
- }
+async function init() {
+ await dbcheck();
- knex.migrate.latest() // And now the current migration with Knex
+ await knex.migrate.latest(); // And now the current migration with Knex
- .then(() => shares.regenerateRoleNamesTable())
- .then(() => shares.rebuildPermissions())
+ await shares.regenerateRoleNamesTable();
+ await shares.rebuildPermissions();
-/* Simplified startup without services - only for debugging the UI and models
- .then(() =>
- startHTTPServer(AppType.TRUSTED, 'trusted', trustedPort, () =>
- startHTTPServer(AppType.SANDBOXED, 'sandbox', sandboxPort, () =>
- startHTTPServer(AppType.PUBLIC, 'public', publicPort, async () => {
+ /*
+ await executor.spawn();
+ await testServer.spawn();
+ await verpServer.spawn();
+ await builtinZoneMta.spawn();
+ */
- await privilegeHelpers.ensureMailtrainDir(uploadedFilesDir);
+ await startHTTPServer(AppType.TRUSTED, 'trusted', trustedPort);
+ await startHTTPServer(AppType.SANDBOXED, 'sandbox', sandboxPort);
+ await startHTTPServer(AppType.PUBLIC, 'public', publicPort);
- privilegeHelpers.dropRootPrivileges();
+ await privilegeHelpers.ensureMailtrainDir(filesDir);
+ await privilegeHelpers.ensureMailtrainDir(uploadedFilesDir);
+ await privilegeHelpers.ensureMailtrainDir(reportFilesDir);
- tzupdate.start();
+ privilegeHelpers.dropRootPrivileges();
- log.info('Service', 'All services started');
- appBuilder.setReady();
- })
- )
- )
- );
-*/
+ /*
+ tzupdate.start();
- .then(() =>
- executor.spawn(() =>
- testServer(() =>
- verpServer(() =>
- builtinZoneMta.spawn(() =>
- startHTTPServer(AppType.TRUSTED, 'trusted', trustedPort, () =>
- startHTTPServer(AppType.SANDBOXED, 'sandbox', sandboxPort, () =>
- startHTTPServer(AppType.PUBLIC, 'public', publicPort, async () => {
+ await importer.spawn();
+ await feedcheck.spawn();
+ await senders.spawn();
- await privilegeHelpers.ensureMailtrainDir(filesDir);
- await privilegeHelpers.ensureMailtrainDir(uploadedFilesDir);
+ triggers.start();
+ gdprCleanup.start();
- privilegeHelpers.dropRootPrivileges();
+ await postfixBounceServer.spawn();
- tzupdate.start();
+ await reportProcessor.init();
+ */
- importer.spawn(() =>
- feedcheck.spawn(() =>
- senders.spawn(() => {
- triggers.start();
- gdprCleanup.start();
+ log.info('Service', 'All services started');
+ appBuilder.setReady();
+}
- postfixBounceServer(async () => {
- await reportProcessor.init();
- log.info('Service', 'All services started');
- appBuilder.setReady();
- });
- })
- )
- );
- })
- )
- )
- )
- )
- )
- )
- );
-});
+init().catch(err => {log.error('', err); process.exit(1); });
diff --git a/server/lib/builtin-zone-mta.js b/server/lib/builtin-zone-mta.js
index 5214e660..53e31ccc 100644
--- a/server/lib/builtin-zone-mta.js
+++ b/server/lib/builtin-zone-mta.js
@@ -6,6 +6,7 @@ const log = require('./log');
const path = require('path');
const fs = require('fs-extra')
const crypto = require('crypto');
+const bluebird = require('bluebird');
let zoneMtaProcess;
@@ -155,6 +156,6 @@ function spawn(callback) {
}
}
-module.exports.spawn = spawn;
+module.exports.spawn = bluebird.promisify(spawn);
module.exports.getUsername = getUsername;
module.exports.getPassword = getPassword;
diff --git a/server/lib/dbcheck.js b/server/lib/dbcheck.js
index eeb1b999..86dbafaa 100644
--- a/server/lib/dbcheck.js
+++ b/server/lib/dbcheck.js
@@ -10,6 +10,7 @@ const log = require('./log');
const fs = require('fs');
const pathlib = require('path');
const Handlebars = require('handlebars');
+const bluebird = require('bluebird');
const highestLegacySchemaVersion = 33;
@@ -136,8 +137,30 @@ function runInitial(callback) {
}, callback);
}
-function runUpdates(callback, runCount) {
- runCount = Number(runCount) || 0;
+function applyUpdate(update, callback) {
+ getSql(update.path, update.data, (err, sql) => {
+ if (err) {
+ return callback(err);
+ }
+
+ db.getConnection((err, connection) => {
+ if (err) {
+ return callback(err);
+ }
+
+ connection.query(sql, err => {
+ connection.release();
+ if (err) {
+ return callback(err);
+ }
+
+ return callback(null, true);
+ });
+ });
+ });
+}
+
+function runUpdates(runCount, callback) {
listTables((err, tables) => {
if (err) {
return callback(err);
@@ -148,7 +171,7 @@ function runUpdates(callback, runCount) {
return callback(new Error('Settings table not found from database'));
}
log.info('sql', 'SQL not set up, initializing');
- return runInitial(runUpdates.bind(null, callback, ++runCount));
+ return runInitial(runUpdates.bind(null, ++runCount, callback));
}
getSchemaVersion((err, schemaVersion) => {
@@ -196,37 +219,13 @@ function runUpdates(callback, runCount) {
});
}
-function applyUpdate(update, callback) {
- getSql(update.path, update.data, (err, sql) => {
- if (err) {
- return callback(err);
- }
+const runUpdatesAsync = bluebird.promisify(runUpdates);
+const dbEndAsync = bluebird.promisify(db.end.bind(db));
- db.getConnection((err, connection) => {
- if (err) {
- return callback(err);
- }
-
- connection.query(sql, err => {
- connection.release();
- if (err) {
- return callback(err);
- }
-
- return callback(null, true);
- });
- });
- });
+async function dbcheck() {
+ await runUpdatesAsync(0);
+ await dbEndAsync();
+ log.info('sql', 'Database check completed');
}
-module.exports = callback => {
- runUpdates(err => {
- if (err) {
- return callback(err);
- }
- db.end(() => {
- log.info('sql', 'Database check completed');
- return callback(null, true);
- });
- });
-};
+module.exports = dbcheck;
diff --git a/server/lib/executor.js b/server/lib/executor.js
index 1a40bd15..c347e6fb 100644
--- a/server/lib/executor.js
+++ b/server/lib/executor.js
@@ -3,17 +3,12 @@
const fork = require('child_process').fork;
const log = require('./log');
const path = require('path');
+const bluebird = require('bluebird');
const requestCallbacks = {};
let messageTid = 0;
let executorProcess;
-module.exports = {
- spawn,
- start,
- stop
-};
-
function spawn(callback) {
log.verbose('Executor', 'Spawning executor process');
@@ -81,3 +76,6 @@ function stop(tid) {
});
}
+module.exports.spawn = bluebird.promisify(spawn);
+module.exports.start = start;
+module.exports.stop = stop;
diff --git a/server/lib/feedcheck.js b/server/lib/feedcheck.js
index f2839eed..c8cb80d4 100644
--- a/server/lib/feedcheck.js
+++ b/server/lib/feedcheck.js
@@ -4,15 +4,11 @@ const fork = require('child_process').fork;
const log = require('./log');
const path = require('path');
const senders = require('./senders');
+const bluebird = require('bluebird');
let messageTid = 0;
let feedcheckProcess;
-module.exports = {
- spawn,
- scheduleCheck
-};
-
function spawn(callback) {
log.verbose('Feed', 'Spawning feedcheck process');
@@ -46,3 +42,5 @@ function scheduleCheck() {
messageTid++;
}
+module.exports.spawn = bluebird.promisify(spawn);
+module.exports.scheduleCheck = scheduleCheck;
\ No newline at end of file
diff --git a/server/lib/file-cache.js b/server/lib/file-cache.js
new file mode 100644
index 00000000..981d40e9
--- /dev/null
+++ b/server/lib/file-cache.js
@@ -0,0 +1,166 @@
+'use strict';
+
+const { filesDir } = require('../models/files');
+const path = require('path');
+const fs = require('fs-extra-promise');
+const stream = require('stream');
+const privilegeHelpers = require('./privilege-helpers');
+const synchronized = require('./synchronized');
+const { tmpName } = require('tmp-promise');
+
+const fileCacheFilesDir = path.join(filesDir, 'cache');
+
+const fileCaches = new Map();
+
+async function _fileCache(typeId, cacheConfig, fileNameGen) {
+ if (fileCaches.has(typeId)) {
+ return fileCaches.get(typeId);
+ }
+
+ const localFilesDir = path.join(fileCacheFilesDir, typeId);
+ await fs.emptyDirAsync(localFilesDir);
+ await privilegeHelpers.ensureMailtrainDir(localFilesDir);
+
+ const cachedFiles = new Map();
+ let nextFilesOrder = 1;
+
+ const pruneCache = async() => {
+ const entries = [];
+ for (const entry of cachedFiles.values()) {
+ if (entry.isReady) {
+ entries.push(entry);
+ }
+ }
+
+ entries.sort((x, y) => y.order - x.order);
+
+ let cumulativeSize = 0;
+ const maxSize = cacheConfig.maxSize * 1048576;
+ for (const entry of entries) {
+ cumulativeSize += entry.size;
+ if (cumulativeSize > maxSize) {
+ entry.isReady = false;
+ await fs.unlinkAsync(path.join(localFilesDir, entry.fileName));
+ cachedFiles.delete(entry.fileName);
+ }
+ }
+ };
+
+ const thisFileCache = (req, res, next) => {
+ const fileName = fileNameGen ? fileNameGen(req) : req.url.substring(1);
+ const localFilePath = path.join(localFilesDir, fileName);
+
+ const fileInfo = cachedFiles.get(fileName);
+ if (fileInfo && fileInfo.isReady) {
+ res.sendFile(
+ localFilePath,
+ {
+ headers: fileInfo.headers
+ },
+ err => {
+ if (err) next(err);
+ }
+ );
+
+ } else {
+ // This means that the file is not present. We thus generate it and cache it.
+ let fileStream = null;
+ let tmpFilePath = null;
+
+ // If the file does not exist yet, we store. If we receive a simulataneous request, while the file is being generate and stored,
+ // we only generate it (but not store it) in the second parallel request.
+ const isStoring = !fileInfo;
+ if (isStoring) {
+ cachedFiles.set(fileName, {
+ fileName,
+ isReady: false
+ });
+ }
+
+ const ensureFileStream = callback => {
+ if (!fileStream) {
+ tmpName().then(tmp => {
+ tmpFilePath = tmp;
+ fileStream = fs.createWriteStream(tmpFilePath);
+ callback();
+ })
+ } else {
+ callback();
+ }
+ };
+
+ let fileSize = 0;
+
+ res.fileCacheResponse = new stream.Writable({
+ write(chunk, encoding, callback) {
+ res.write(chunk, encoding);
+
+ if (isStoring) {
+ fileSize += chunk.length;
+ ensureFileStream(() => {
+ fileStream.write(chunk, encoding);
+ callback();
+ });
+ } else {
+ callback();
+ }
+ },
+
+ final(callback) {
+ res.end();
+
+ if (isStoring) {
+ ensureFileStream(() => {
+ fileStream.end(null, null, () => {
+ fs.moveAsync(tmpFilePath, localFilePath, {})
+ .then(() => {
+ cachedFiles.set(fileName, {
+ fileName,
+ size: fileSize,
+ order: nextFilesOrder,
+ headers: res.getHeaders(),
+ isReady: true
+ });
+
+ nextFilesOrder += 1;
+
+ callback();
+
+ // noinspection JSIgnoredPromiseFromCall
+ pruneCache();
+ })
+ .catch(err => next(err));
+ });
+ });
+ } else {
+ callback();
+ }
+ },
+
+ destroy(err, callback) {
+ res.destroy(err);
+
+ if (fileStream) {
+ fileStream.destroy(err);
+ fs.unlink(tmpFilePath, () => {
+ cachedFiles.delete(fileName);
+ callback();
+ });
+ } else {
+ callback();
+ }
+ }
+ });
+
+ next();
+ }
+ };
+
+ fileCaches.set(typeId, thisFileCache);
+ return thisFileCache;
+}
+
+const fileCache = synchronized(_fileCache);
+
+module.exports.fileCache = fileCache;
+module.exports.fileCacheFilesDir = fileCacheFilesDir;
\ No newline at end of file
diff --git a/server/lib/importer.js b/server/lib/importer.js
index 5e59dd64..11576633 100644
--- a/server/lib/importer.js
+++ b/server/lib/importer.js
@@ -7,15 +7,11 @@ const path = require('path');
const {ImportStatus, RunStatus} = require('../../shared/imports');
const {ListActivityType} = require('../../shared/activity-log');
const activityLog = require('./activity-log');
+const bluebird = require('bluebird');
let messageTid = 0;
let importerProcess;
-module.exports = {
- spawn,
- scheduleCheck
-};
-
function spawn(callback) {
log.verbose('Importer', 'Spawning importer process');
@@ -65,4 +61,5 @@ function scheduleCheck() {
messageTid++;
}
-
+module.exports.spawn = bluebird.promisify(spawn);
+module.exports.scheduleCheck = scheduleCheck;
\ No newline at end of file
diff --git a/server/lib/privilege-helpers.js b/server/lib/privilege-helpers.js
index d1f4671c..9bc5c0da 100644
--- a/server/lib/privilege-helpers.js
+++ b/server/lib/privilege-helpers.js
@@ -53,7 +53,7 @@ function ensureMailtrainOwner(file, callback) {
fs.chown(file, ids.uid, ids.gid, callback);
}
-async function ensureMailtrainDir(dir) {
+async function ensureMailtrainDir(dir, recursive) {
const ids = getConfigUidGid();
await fs.ensureDir(dir);
await fs.chownAsync(dir, ids.uid, ids.gid);
diff --git a/server/lib/report-helpers.js b/server/lib/report-helpers.js
index deee31e1..b2d3da43 100644
--- a/server/lib/report-helpers.js
+++ b/server/lib/report-helpers.js
@@ -11,9 +11,10 @@ function nameToFileName(name) {
replace(/--*/g, '-');
}
+const reportFilesDir = path.join(__dirname, '..', 'files', 'reports');
function getReportFileBase(report) {
- return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + nameToFileName(report.name));
+ return path.join(reportFilesDir, report.id + '-' + nameToFileName(report.name));
}
function getReportContentFile(report) {
@@ -28,5 +29,6 @@ function getReportOutputFile(report) {
module.exports = {
getReportContentFile,
getReportOutputFile,
- nameToFileName
+ nameToFileName,
+ reportFilesDir
};
diff --git a/server/lib/senders.js b/server/lib/senders.js
index 60e07f82..da48803f 100644
--- a/server/lib/senders.js
+++ b/server/lib/senders.js
@@ -6,6 +6,7 @@ const path = require('path');
const knex = require('./knex');
const {CampaignStatus} = require('../../shared/campaigns');
const builtinZoneMta = require('./builtin-zone-mta');
+const bluebird = require('bluebird');
let messageTid = 0;
let senderProcess;
@@ -59,9 +60,6 @@ function reloadConfig(sendConfigurationId) {
messageTid++;
}
-module.exports = {
- spawn,
- scheduleCheck,
- reloadConfig
-};
-
+module.exports.spawn = bluebird.promisify(spawn);
+module.exports.scheduleCheck = scheduleCheck;
+module.exports.reloadConfig = reloadConfig;
\ No newline at end of file
diff --git a/server/lib/synchronized.js b/server/lib/synchronized.js
new file mode 100644
index 00000000..16c853a1
--- /dev/null
+++ b/server/lib/synchronized.js
@@ -0,0 +1,26 @@
+'use strict';
+
+// This implements a simple wrapper around an async function that prevents concurrent execution of the function from two asynchronous chains
+// It enforces that the running execution has to complete first before another one is started.
+function synchronized(asyncFn) {
+ let ensurePromise = null;
+
+ return async (...args) => {
+ while (ensurePromise) {
+ try {
+ await ensurePromise;
+ } catch (err) {
+ }
+ }
+
+ ensurePromise = asyncFn(...args);
+
+ try {
+ return await ensurePromise;
+ } finally {
+ ensurePromise = null;
+ }
+ }
+}
+
+module.exports = synchronized;
\ No newline at end of file
diff --git a/server/models/files.js b/server/models/files.js
index 6dce0aa0..2ceaf928 100644
--- a/server/models/files.js
+++ b/server/models/files.js
@@ -12,7 +12,7 @@ const {getPublicUrl} = require('../lib/urls');
const crypto = require('crypto');
const bluebird = require('bluebird');
-const cryptoPseudoRandomBytes = bluebird.promisify(crypto.pseudoRandomBytes);
+const cryptoPseudoRandomBytes = bluebird.promisify(crypto.pseudoRandomBytes.bind(crypto));
const entityTypes = entitySettings.getEntityTypes();
diff --git a/server/models/reports.js b/server/models/reports.js
index 8be58e3b..992ae234 100644
--- a/server/models/reports.js
+++ b/server/models/reports.js
@@ -155,7 +155,6 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
let firstIteration = true;
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
- const subsTable = subscriptions.getSubscriptionTableName(cpgListId);
const flds = await fields.list(contextHelpers.getAdminContext(), cpgListId);
@@ -184,7 +183,6 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
for (const cpgList of campaign.lists) {
const cpgListId = cpgList.list;
- const subsTable = subscriptions.getSubscriptionTableName(cpgListId);
const campaignFieldsMapping = {
'list:id': {raw: knex.raw('?', [cpgListId])},
@@ -203,6 +201,14 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
...campaignFieldsMapping
};
+ const getColIdIfExists = (colId, getter) => {
+ if (colId in fieldsMapping) {
+ return getter(colId);
+ } else {
+ throw new Error(`Unknown column id ${colId}`);
+ }
+ }
+
const getSelField = item => {
const itemMapping = fieldsMapping[item];
if (typeof itemMapping === 'string') {
@@ -226,17 +232,22 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
let query = knex(`subscription__${cpgListId} AS subscriptions`)
.leftJoin('campaign_messages', {
- 'campaign_messages.subscription': 'subscriptions.id',
- 'campaign_messages.list': knex.raw('?', [cpgListId])
+ 'campaign_messages.campaign': knex.raw('?', [campaign.id]),
+ 'campaign_messages.list': knex.raw('?', [cpgListId]),
+ 'campaign_messages.subscription': 'subscriptions.id'
})
.leftJoin('campaign_links', {
- 'campaign_links.subscription': 'subscriptions.id',
- 'campaign_links.list': knex.raw('?', [cpgListId])
+ 'campaign_links.campaign': knex.raw('?', [campaign.id]),
+ 'campaign_links.list': knex.raw('?', [cpgListId]),
+ 'campaign_links.subscription': 'subscriptions.id'
})
.select(selFields);
if (listQryFn) {
- query = listQryFn(query);
+ query = listQryFn(
+ query,
+ colId => getColIdIfExists(colId, x => fieldsMapping[x])
+ );
}
subsQrys.push(query.toSQL().toNative());
@@ -250,7 +261,8 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
return unionQryFn(
knex.from(function() {
return knex.raw('(' + subsSql + ')', subsBindings);
- })
+ }),
+ colId => getColIdIfExists(colId, x => x)
);
} else {
return knex.raw(subsSql, subsBindings);
@@ -267,6 +279,7 @@ async function _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, a
if (asStream) {
return applyUnionQryFn(subsSql, subsBindings).stream();
+
} else {
const res = await applyUnionQryFn(subsSql, subsBindings);
if (res[0] && Array.isArray(res[0])) {
@@ -299,10 +312,11 @@ async function _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryF
campaign,
select,
unionQryFn,
- qry => listQryFn(
+ (qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.OPEN)
- })
+ }),
+ col
),
asStream
);
@@ -317,10 +331,11 @@ async function _getCampaignClickStatistics(campaign, select, unionQryFn, listQry
campaign,
select,
unionQryFn,
- qry => listQryFn(
+ (qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', LinkId.GENERAL_CLICK)
- })
+ }),
+ col
),
asStream
);
@@ -335,15 +350,24 @@ async function _getCampaignLinkClickStatistics(campaign, select, unionQryFn, lis
campaign,
select,
unionQryFn,
- qry => listQryFn(
+ (qry, col) => listQryFn(
qry.where(function() {
this.whereNull('campaign_links.link').orWhere('campaign_links.link', '>', LinkId.GENERAL_CLICK)
- })
+ }),
+ col
),
asStream
);
}
+async function getCampaignStatistics(campaign, select, unionQryFn, listQryFn) {
+ return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, false);
+}
+
+async function getCampaignStatisticsStream(campaign, select, unionQryFn, listQryFn) {
+ return await _getCampaignStatistics(campaign, select, unionQryFn, listQryFn, true);
+}
+
async function getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn) {
return await _getCampaignOpenStatistics(campaign, select, unionQryFn, listQryFn, false);
}
@@ -381,6 +405,8 @@ module.exports.remove = remove;
module.exports.updateFields = updateFields;
module.exports.listByState = listByState;
module.exports.bulkChangeState = bulkChangeState;
+module.exports.getCampaignStatistics = getCampaignStatistics;
+module.exports.getCampaignStatisticsStream = getCampaignStatisticsStream;
module.exports.getCampaignOpenStatistics = getCampaignOpenStatistics;
module.exports.getCampaignClickStatistics = getCampaignClickStatistics;
module.exports.getCampaignLinkClickStatistics = getCampaignLinkClickStatistics;
diff --git a/server/models/users.js b/server/models/users.js
index 0aee522b..5b7da1bd 100644
--- a/server/models/users.js
+++ b/server/models/users.js
@@ -16,8 +16,8 @@ const { tUI } = require('../lib/translate');
const bluebird = require('bluebird');
const bcrypt = require('bcrypt-nodejs');
-const bcryptHash = bluebird.promisify(bcrypt.hash);
-const bcryptCompare = bluebird.promisify(bcrypt.compare);
+const bcryptHash = bluebird.promisify(bcrypt.hash.bind(bcrypt));
+const bcryptCompare = bluebird.promisify(bcrypt.compare.bind(bcrypt));
const mailers = require('../lib/mailers');
diff --git a/server/package-lock.json b/server/package-lock.json
index 2467d264..56ff4b78 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -575,9 +575,9 @@
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
},
"aws-sdk": {
- "version": "2.437.0",
- "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.437.0.tgz",
- "integrity": "sha512-sDZb5QBOO6FOMvuKDEdO16YQRk0WUhnQd38EaSt0yUCi4Gev8uypODyYONgODZcXe8Cr1GMwC8scUKr00S/I5w==",
+ "version": "2.440.0",
+ "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.440.0.tgz",
+ "integrity": "sha512-/sXI7pZggvFMr2J9qCpGgE2XO/4uBErRVSGoHTR3PkGWvf352w+HebnuGdRKK9D3lnGuGMnEY8w9IS44LUKsxw==",
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
@@ -722,9 +722,9 @@
}
},
"bignumber.js": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.1.0.tgz",
- "integrity": "sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA=="
+ "version": "7.2.1",
+ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz",
+ "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ=="
},
"binary": {
"version": "0.3.0",
@@ -1223,24 +1223,6 @@
"vary": "~1.1.2"
}
},
- "compressjs": {
- "version": "github:openpgpjs/compressjs#bfbb371a34d1750afa34bfa49156461acdab79a9",
- "from": "github:openpgpjs/compressjs",
- "requires": {
- "amdefine": "~1.0.0",
- "commander": "~2.8.1"
- },
- "dependencies": {
- "commander": {
- "version": "2.8.1",
- "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz",
- "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=",
- "requires": {
- "graceful-readlink": ">= 1.0.0"
- }
- }
- }
- },
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -1477,9 +1459,9 @@
}
},
"csv-parse": {
- "version": "4.4.0",
- "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.4.0.tgz",
- "integrity": "sha512-rvoRlZxu6Ap8jOkhoQQeI+5y/eTPqEIVk20bxZmo81k2ArUiNLv8LAERTEKarOQuC7BKXGyzSqAKWox115bg7A=="
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.4.1.tgz",
+ "integrity": "sha512-uFe5phPfmwBXSPWz5GYHeaEc2Oezn2kY5iLIvG1sJjc32Y4GU7T/b/uX5ffZh4CBDWwJQjwAuxrDEdl3Z5Qv+g=="
},
"csv-stringify": {
"version": "5.3.0",
@@ -1768,19 +1750,6 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
},
- "elliptic": {
- "version": "github:openpgpjs/elliptic#ad81845f693effa5b4b6d07db2e82112de222f48",
- "from": "github:openpgpjs/elliptic",
- "requires": {
- "bn.js": "^4.4.0",
- "brorand": "^1.0.1",
- "hash.js": "^1.0.0",
- "hmac-drbg": "^1.0.0",
- "inherits": "^2.0.1",
- "minimalistic-assert": "^1.0.0",
- "minimalistic-crypto-utils": "^1.0.0"
- }
- },
"email-addresses": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/email-addresses/-/email-addresses-3.0.3.tgz",
@@ -5602,9 +5571,9 @@
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
},
"moment-timezone": {
- "version": "0.5.23",
- "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.23.tgz",
- "integrity": "sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==",
+ "version": "0.5.25",
+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.25.tgz",
+ "integrity": "sha512-DgEaTyN/z0HFaVcVbSyVCUU6HeFdnNC3vE4c9cgu2dgMTvjBUBdBzWfasTBmAW45u5OIMeCJtU8yNjM22DHucw==",
"requires": {
"moment": ">= 2.9.0"
}
@@ -5652,11 +5621,11 @@
"integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s="
},
"mysql": {
- "version": "2.16.0",
- "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.16.0.tgz",
- "integrity": "sha512-dPbN2LHonQp7D5ja5DJXNbCLe/HRdu+f3v61aguzNRQIrmZLOeRoymBYyeThrR6ug+FqzDL95Gc9maqZUJS+Gw==",
+ "version": "2.17.1",
+ "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.17.1.tgz",
+ "integrity": "sha512-7vMqHQ673SAk5C8fOzTG2LpPcf3bNt0oL3sFpxPEEFp1mdlDcrLK0On7z8ZYKaaHrHwNcQ/MTUz7/oobZ2OyyA==",
"requires": {
- "bignumber.js": "4.1.0",
+ "bignumber.js": "7.2.1",
"readable-stream": "2.3.6",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
@@ -5816,6 +5785,35 @@
"ieee754": "^1.1.4"
}
},
+ "commander": {
+ "version": "2.8.1",
+ "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz",
+ "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=",
+ "requires": {
+ "graceful-readlink": ">= 1.0.0"
+ }
+ },
+ "compressjs": {
+ "version": "github:openpgpjs/compressjs#bfbb371a34d1750afa34bfa49156461acdab79a9",
+ "from": "github:openpgpjs/compressjs#bfbb371a34d1750afa34bfa49156461acdab79a9",
+ "requires": {
+ "amdefine": "~1.0.0",
+ "commander": "~2.8.1"
+ }
+ },
+ "elliptic": {
+ "version": "github:openpgpjs/elliptic#ad81845f693effa5b4b6d07db2e82112de222f48",
+ "from": "github:openpgpjs/elliptic#ad81845f693effa5b4b6d07db2e82112de222f48",
+ "requires": {
+ "bn.js": "^4.4.0",
+ "brorand": "^1.0.1",
+ "hash.js": "^1.0.0",
+ "hmac-drbg": "^1.0.0",
+ "inherits": "^2.0.1",
+ "minimalistic-assert": "^1.0.0",
+ "minimalistic-crypto-utils": "^1.0.0"
+ }
+ },
"openpgp": {
"version": "3.0.11",
"resolved": "https://registry.npmjs.org/openpgp/-/openpgp-3.0.11.tgz",
@@ -5833,6 +5831,29 @@
"node-localstorage": "~1.3.0",
"pako": "^1.0.6",
"rusha": "^0.8.12"
+ },
+ "dependencies": {
+ "compressjs": {
+ "version": "github:openpgpjs/compressjs#bfbb371a34d1750afa34bfa49156461acdab79a9",
+ "from": "github:openpgpjs/compressjs",
+ "requires": {
+ "amdefine": "~1.0.0",
+ "commander": "~2.8.1"
+ }
+ },
+ "elliptic": {
+ "version": "github:openpgpjs/elliptic#ad81845f693effa5b4b6d07db2e82112de222f48",
+ "from": "github:openpgpjs/elliptic",
+ "requires": {
+ "bn.js": "^4.4.0",
+ "brorand": "^1.0.1",
+ "hash.js": "^1.0.0",
+ "hmac-drbg": "^1.0.0",
+ "inherits": "^2.0.1",
+ "minimalistic-assert": "^1.0.0",
+ "minimalistic-crypto-utils": "^1.0.0"
+ }
+ }
}
}
}
@@ -7511,6 +7532,15 @@
"os-tmpdir": "~1.0.2"
}
},
+ "tmp-promise": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-1.0.5.tgz",
+ "integrity": "sha512-hOabTz9Tp49wCozFwuJe5ISrOqkECm6kzw66XTP23DuzNU7QS/KiZq5LC9Y7QSy8f1rPSLy4bKaViP0OwGI1cA==",
+ "requires": {
+ "bluebird": "^3.5.0",
+ "tmp": "0.0.33"
+ }
+ },
"to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
diff --git a/server/package.json b/server/package.json
index 25d7230a..b715a580 100644
--- a/server/package.json
+++ b/server/package.json
@@ -42,7 +42,7 @@
"posix": "^4.1.2"
},
"dependencies": {
- "aws-sdk": "^2.383.0",
+ "aws-sdk": "^2.440.0",
"bcrypt-nodejs": "0.0.3",
"bluebird": "^3.5.3",
"body-parser": "^1.18.3",
@@ -56,7 +56,7 @@
"cors": "^2.8.5",
"crypto": "^1.0.1",
"csurf": "^1.9.0",
- "csv-parse": "^4.3.0",
+ "csv-parse": "^4.4.1",
"csv-stringify": "^5.1.2",
"device": "^0.3.9",
"dompurify": "^1.0.8",
@@ -80,16 +80,16 @@
"isemail": "^3.2.0",
"jsdom": "^13.1.0",
"juice": "^5.1.0",
- "knex": "^0.16.3",
+ "knex": "^0.16.5",
"libmime": "^4.0.1",
"mailparser": "^2.4.3",
"memory-cache": "^0.2.0",
"mjml": "^4.3.0",
"moment": "^2.23.0",
- "moment-timezone": "^0.5.23",
+ "moment-timezone": "^0.5.25",
"morgan": "^1.9.1",
"multer": "^1.4.1",
- "mysql": "^2.16.0",
+ "mysql": "^2.17.1",
"node-ipc": "^9.1.1",
"node-mocks-http": "^1.7.3",
"node-object-hash": "^1.4.1",
@@ -107,6 +107,7 @@
"shortid": "^2.2.14",
"slugify": "^1.3.4",
"smtp-server": "^3.4.7",
+ "tmp-promise": "^1.0.5",
"toml": "^2.3.3",
"try-require": "^1.2.1",
"xmldom": "^0.1.27"
diff --git a/server/routes/index.js b/server/routes/index.js
index 97eebccf..62e2abf6 100644
--- a/server/routes/index.js
+++ b/server/routes/index.js
@@ -7,7 +7,7 @@ const { AppType } = require('../../shared/app');
const routerFactory = require('../lib/router-async');
-function getRouter(appType) {
+async function getRouter(appType) {
const router = routerFactory.create();
if (appType === AppType.TRUSTED) {
diff --git a/server/routes/rest/editors.js b/server/routes/rest/editors.js
index 96baf0cf..0fc5bdb3 100644
--- a/server/routes/rest/editors.js
+++ b/server/routes/rest/editors.js
@@ -4,7 +4,7 @@ const passport = require('../../lib/passport');
const bluebird = require('bluebird');
const premailerApi = require('premailer-api');
-const premailerPrepareAsync = bluebird.promisify(premailerApi.prepare);
+const premailerPrepareAsync = bluebird.promisify(premailerApi.prepare.bind(premailerApi));
const router = require('../../lib/router-async').create();
diff --git a/server/routes/sandboxed-ckeditor.js b/server/routes/sandboxed-ckeditor.js
index 02c98c3a..77852e96 100644
--- a/server/routes/sandboxed-ckeditor.js
+++ b/server/routes/sandboxed-ckeditor.js
@@ -33,7 +33,7 @@ users.registerRestrictedAccessTokenMethod('ckeditor', async ({entityTypeId, enti
});
-function getRouter(appType) {
+async function getRouter(appType) {
const router = routerFactory.create();
if (appType === AppType.SANDBOXED) {
diff --git a/server/routes/sandboxed-codeeditor.js b/server/routes/sandboxed-codeeditor.js
index e8d926fc..6d128412 100644
--- a/server/routes/sandboxed-codeeditor.js
+++ b/server/routes/sandboxed-codeeditor.js
@@ -33,7 +33,7 @@ users.registerRestrictedAccessTokenMethod('codeeditor', async ({entityTypeId, en
});
-function getRouter(appType) {
+async function getRouter(appType) {
const router = routerFactory.create();
if (appType === AppType.SANDBOXED) {
diff --git a/server/routes/sandboxed-grapesjs.js b/server/routes/sandboxed-grapesjs.js
index f4a39b9b..55ce4162 100644
--- a/server/routes/sandboxed-grapesjs.js
+++ b/server/routes/sandboxed-grapesjs.js
@@ -33,7 +33,7 @@ users.registerRestrictedAccessTokenMethod('grapesjs', async ({entityTypeId, enti
});
-function getRouter(appType) {
+async function getRouter(appType) {
const router = routerFactory.create();
if (appType === AppType.SANDBOXED) {
diff --git a/server/routes/sandboxed-mosaico.js b/server/routes/sandboxed-mosaico.js
index 610d4018..7112cfef 100644
--- a/server/routes/sandboxed-mosaico.js
+++ b/server/routes/sandboxed-mosaico.js
@@ -31,6 +31,8 @@ const { AppType } = require('../../shared/app');
const {castToInteger} = require('../lib/helpers');
+const { fileCache } = require('../lib/file-cache');
+
users.registerRestrictedAccessTokenMethod('mosaico', async ({entityTypeId, entityId}) => {
if (entityTypeId === 'template') {
@@ -131,7 +133,7 @@ function sanitizeSize(val, min, max, defaultVal, allowNull) {
-function getRouter(appType) {
+async function getRouter(appType) {
const router = routerFactory.create();
if (appType === AppType.SANDBOXED) {
@@ -161,14 +163,14 @@ function getRouter(appType) {
router.use('/templates/:mosaicoTemplateId/edres', express.static(path.join(__dirname, '..', '..', 'client', 'static', 'mosaico', 'templates', 'versafix-1', 'edres')));
// This is the final fallback for a block thumbnail, so that at least something gets returned
- router.getAsync('/templates/:mosaicoTemplateId/edres/:fileName', async (req, res, next) => {
+ router.getAsync('/templates/:mosaicoTemplateId/edres/:fileName', await fileCache('mosaico-block-thumbnails', config.mosaico.fileCache.blockThumbnails, req => req.params.fileName), async (req, res) => {
let labelText = req.params.fileName.replace(/\.png$/, '');
labelText = labelText.replace(/[_]/g, ' ');
labelText = capitalize.words(labelText);
const image = await placeholderImage(340, 100, labelText, '#ffffff');
res.set('Content-Type', 'image/' + image.format);
- image.stream.pipe(res);
+ image.stream.pipe(res.fileCacheResponse);
});
fileHelpers.installUploadHandler(router, '/upload/:type/:entityId', files.ReplacementBehavior.RENAME, null, 'file', resp => {
@@ -225,7 +227,7 @@ function getRouter(appType) {
} else if (appType === AppType.TRUSTED || appType === AppType.PUBLIC) { // Mosaico editor loads the images from TRUSTED endpoint. This is hard to change because the index.html has to come from TRUSTED.
// So we serve /mosaico/img under both endpoints. There is no harm in it.
- router.getAsync('/img', async (req, res) => {
+ router.getAsync('/img', await fileCache('mosaico-images', config.mosaico.fileCache.images), async (req, res) => {
const method = req.query.method;
const params = req.query.params;
let [width, height] = params.split(',');
@@ -248,7 +250,7 @@ function getRouter(appType) {
height = sanitizeSize(height, 1, 2048, 300, true);
let filePath;
- const url = req.query.src;
+ const url = req.query.src || '';
const mosaicoLegacyUrlPrefix = getTrustedUrl(`mosaico/uploads/`);
if (url.startsWith(mosaicoLegacyUrlPrefix)) {
@@ -262,7 +264,7 @@ function getRouter(appType) {
}
res.set('Content-Type', 'image/' + image.format);
- image.stream.pipe(res);
+ image.stream.pipe(res.fileCacheResponse);
});
}
diff --git a/server/routes/subscription.js b/server/routes/subscription.js
index 9ed8fd80..258f3164 100644
--- a/server/routes/subscription.js
+++ b/server/routes/subscription.js
@@ -334,7 +334,7 @@ router.getAsync('/:cid/widget', cors(corsOptions), async (req, res) => {
await injectCustomFormData(req.query.fid || list.default_form, 'web_subscribe', data);
- const renderAsync = bluebird.promisify(res.render);
+ const renderAsync = bluebird.promisify(res.render.bind(res));
const html = await renderAsync('subscription/widget-subscribe', data);
const response = {
diff --git a/server/services/postfix-bounce-server.js b/server/services/postfix-bounce-server.js
index 9488a3e1..4a39a888 100644
--- a/server/services/postfix-bounce-server.js
+++ b/server/services/postfix-bounce-server.js
@@ -6,6 +6,7 @@ const net = require('net');
const campaigns = require('../models/campaigns');
const contextHelpers = require('../lib/context-helpers');
const { SubscriptionStatus } = require('../../shared/lists');
+const bluebird = require('bluebird');
const seenIds = new Set();
@@ -33,7 +34,7 @@ async function readNextChunks() {
try {
const match = /\bstatus=(bounced|sent)\b/.test(line) && line.match(/\bpostfix\/\w+\[\d+\]:\s*([^:]+).*?status=(\w+)/);
if (match) {
- let queueId = match[1];
+ const queueId = match[1];
let queued = '';
let queuedAs = '';
@@ -41,7 +42,7 @@ async function readNextChunks() {
seenIds.add(queueId);
// Losacno: Check for local requeue
- let status = match[2];
+ const status = match[2];
log.verbose('POSTFIXBOUNCE', 'Checking message %s for local requeue (status: %s)', queueId, status);
if (status === 'sent') {
// Save new queueId to update message's previous queueId (thanks @mfechner )
@@ -82,7 +83,7 @@ async function readNextChunks() {
}
}
-module.exports = callback => {
+function spawn(callback) {
if (!config.postfixbounce.enabled) {
return setImmediate(callback);
}
@@ -122,4 +123,7 @@ module.exports = callback => {
log.info('POSTFIXBOUNCE', 'Server listening on port %s', config.postfixbounce.port);
setImmediate(callback);
});
-};
+}
+
+module.exports.spawn = bluebird.promisify(spawn);
+
diff --git a/server/services/test-server.js b/server/services/test-server.js
index 64c6a16e..e6d5ce24 100644
--- a/server/services/test-server.js
+++ b/server/services/test-server.js
@@ -5,6 +5,7 @@ const config = require('config');
const crypto = require('crypto');
const humanize = require('humanize');
const http = require('http');
+const bluebird = require('bluebird');
const SMTPServer = require('smtp-server').SMTPServer;
const simpleParser = require('mailparser').simpleParser;
@@ -22,7 +23,7 @@ const mailstore = {
},
getMail(address, callback) {
if (!this.accounts[address] || this.accounts[address].length === 0) {
- let err = new Error('No mail for ' + address);
+ const err = new Error('No mail for ' + address);
err.status = 404;
return callback(err);
}
@@ -55,8 +56,8 @@ const server = new SMTPServer({
// Setup authentication
onAuth: (auth, session, callback) => {
- let username = config.testServer.username;
- let password = config.testServer.password;
+ const username = config.testServer.username;
+ const password = config.testServer.password;
// check username and password
if (auth.username === username && auth.password === password) {
@@ -80,15 +81,13 @@ const server = new SMTPServer({
// Validate RCPT TO envelope address. Example allows all addresses that do not start with 'deny'
// If this method is not set, all addresses are allowed
onRcptTo: (address, session, callback) => {
- let err;
-
if (/^deny/i.test(address.address)) {
return callback(new Error('Not accepted'));
}
// Reject messages larger than 100 bytes to an over-quota user
if (/^full/i.test(address.address) && Number(session.envelope.mailFrom.args.SIZE) > 100) {
- err = new Error('Insufficient channel storage: ' + address.address);
+ const err = new Error('Insufficient channel storage: ' + address.address);
err.responseCode = 452;
return callback(err);
}
@@ -98,7 +97,7 @@ const server = new SMTPServer({
// Handle message stream
onData: (stream, session, callback) => {
- let hash = crypto.createHash('md5');
+ const hash = crypto.createHash('md5');
let message = '';
stream.on('data', chunk => {
hash.update(chunk);
@@ -107,9 +106,8 @@ const server = new SMTPServer({
}
});
stream.on('end', () => {
- let err;
if (stream.sizeExceeded) {
- err = new Error('Error: message exceeds fixed maximum message size 10 MB');
+ const err = new Error('Error: message exceeds fixed maximum message size 10 MB');
err.responseCode = 552;
return callback(err);
}
@@ -129,15 +127,15 @@ server.on('error', err => {
log.error('Test SMTP', err.stack);
});
-let mailBoxServer = http.createServer((req, res) => {
- let renderer = data => (
+const mailBoxServer = http.createServer((req, res) => {
+ const renderer = data => (
'' + data.title + '' + data.body + ''
);
- let address = req.url.substring(1);
+ const address = req.url.substring(1);
mailstore.getMail(address, (err, mail) => {
if (err) {
- let html = renderer({
+ const html = renderer({
title: 'error',
body: err.message || err
});
@@ -155,7 +153,7 @@ let mailBoxServer = http.createServer((req, res) => {
delete mail.textAsHtml;
delete mail.attachments;
- let script = '';
+ const script = '';
html = html.replace(/<\/body\b/i, match => script + match);
html = html.replace(/target="_blank"/g, 'target="_self"');
@@ -168,7 +166,7 @@ mailBoxServer.on('error', err => {
log.error('Test SMTP Mailbox Server', err);
});
-module.exports = callback => {
+function spawn(callback) {
if (config.testServer.enabled) {
server.listen(config.testServer.port, config.testServer.host, () => {
log.info('Test SMTP', 'Server listening on port %s', config.testServer.port);
@@ -194,4 +192,6 @@ module.exports = callback => {
} else {
setImmediate(callback);
}
-};
+}
+
+module.exports.spawn = bluebird.promisify(spawn);
diff --git a/server/services/verp-server.js b/server/services/verp-server.js
index 9b02aa27..38c79109 100644
--- a/server/services/verp-server.js
+++ b/server/services/verp-server.js
@@ -7,6 +7,7 @@ const {MailerError} = require('../lib/mailers');
const campaigns = require('../models/campaigns');
const contextHelpers = require('../lib/context-helpers');
const {SubscriptionStatus} = require('../../shared/lists');
+const bluebird = require('bluebird');
const BounceHandler = require('bounce-handler').BounceHandler;
const SMTPServer = require('smtp-server').SMTPServer;
@@ -85,7 +86,7 @@ const server = new SMTPServer({
onData: onData
});
-module.exports = callback => {
+function spawn(callback) {
if (!config.verp.enabled) {
return setImmediate(callback);
}
@@ -131,7 +132,7 @@ module.exports = callback => {
started = true;
return setImmediate(callback);
}
- let host = hosts[pos++];
+ const host = hosts[pos++];
server.listen(config.verp.port, host, () => {
if (started) {
return server.close();
@@ -142,4 +143,6 @@ module.exports = callback => {
};
startNextHost();
-};
+}
+
+module.exports.spawn = bluebird.promisify(spawn);
diff --git a/server/services/workers/reports/report-processor.js b/server/services/workers/reports/report-processor.js
index 9097f2c5..b7883032 100644
--- a/server/services/workers/reports/report-processor.js
+++ b/server/services/workers/reports/report-processor.js
@@ -1,9 +1,9 @@
'use strict';
const reports = require('../../../models/reports');
-const reportTemplates = require('../../../models/report-templates');
const lists = require('../../../models/lists');
const subscriptions = require('../../../models/subscriptions');
+const { SubscriptionSource, SubscriptionStatus } = require('../../../../shared/lists');
const campaigns = require('../../../models/campaigns');
const handlebars = require('handlebars');
const hbs = require('hbs');
@@ -50,9 +50,11 @@ async function main() {
}
const campaignsProxy = {
+ getCampaignStatistics: reports.getCampaignStatistics,
getCampaignOpenStatistics: reports.getCampaignOpenStatistics,
getCampaignClickStatistics: reports.getCampaignClickStatistics,
getCampaignLinkClickStatistics: reports.getCampaignLinkClickStatistics,
+ getCampaignStatisticsStream: reports.getCampaignStatisticsStream,
getCampaignOpenStatisticsStream: reports.getCampaignOpenStatisticsStream,
getCampaignClickStatisticsStream: reports.getCampaignClickStatisticsStream,
getCampaignLinkClickStatisticsStream: reports.getCampaignLinkClickStatisticsStream,
@@ -71,17 +73,45 @@ async function main() {
knex,
process,
inputs,
+ SubscriptionSource,
+ SubscriptionStatus,
- renderCsvFromStream: async (readable, opts) => {
- const stringifier = csvStringify(opts);
-
+ renderCsvFromStream: async (readable, opts, transform) => {
const finished = new Promise((success, fail) => {
- stringifier.on('finish', () => success())
- stringifier.on('error', (err) => fail(err))
- });
+ let lastReadable = readable;
- stringifier.pipe(process.stdout);
- readable.pipe(stringifier);
+ const stringifier = csvStringify(opts);
+
+ stringifier.on('finish', () => success());
+ stringifier.on('error', err => fail(err));
+
+ if (transform) {
+ const rowTransform = new stream.Transform({
+ objectMode: true,
+ transform(row, encoding, callback) {
+ async function performTransform() {
+ try {
+ const newRow = await transform(row, encoding);
+ callback(null, newRow);
+ } catch (err) {
+ callback(err);
+ }
+ }
+
+ // noinspection JSIgnoredPromiseFromCall
+ performTransform();
+ }
+ });
+
+ lastReadable.on('error', err => fail(err));
+ lastReadable.pipe(rowTransform);
+
+ lastReadable = rowTransform;
+ }
+
+ stringifier.pipe(process.stdout);
+ lastReadable.pipe(stringifier);
+ });
await finished;
},
diff --git a/setup/functions b/setup/functions
index d3b149f7..b84488cb 100644
--- a/setup/functions
+++ b/setup/functions
@@ -206,7 +206,7 @@ mysql:
password: "$mysqlRoPassword"
EOT
- reinstallModules
+ reinstallAllModules
(cd client && npm run build)
@@ -215,16 +215,25 @@ EOT
}
-function reinstallModules {
+function doForAllModules {
# Install required node packages
for idx in client shared server zone-mta mvis/client mvis/server mvis/test-embed mvis/ivis-core/client mvis/ivis-core/server mvis/ivis-core/shared mvis/ivis-core/embedding; do
if [ -d $idx ]; then
- echo Reinstalling modules in $idx
- (cd $idx && rm -rf node_modules && npm install)
+ ($1 $idx)
fi
done
}
+function reinstallModules {
+ local idx=$1
+ echo Reinstalling modules in $idx
+ cd $idx && rm -rf node_modules && npm install
+}
+
+function reinstallAllModules {
+ doForAllModules reinstallModules
+}
+
function installHttpd {
local portTrusted="$1"
diff --git a/setup/reinstall-modules.sh b/setup/reinstall-modules.sh
index f99567ae..5dd2c626 100644
--- a/setup/reinstall-modules.sh
+++ b/setup/reinstall-modules.sh
@@ -6,4 +6,4 @@ SCRIPT_PATH=$(dirname $(realpath -s $0))
. $SCRIPT_PATH/functions
cd $SCRIPT_PATH/..
-reinstallModules
\ No newline at end of file
+reinstallAllModules
\ No newline at end of file
diff --git a/shared/package-lock.json b/shared/package-lock.json
index d6a4d8ce..338ef5a6 100644
--- a/shared/package-lock.json
+++ b/shared/package-lock.json
@@ -4487,9 +4487,9 @@
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
},
"moment-timezone": {
- "version": "0.5.23",
- "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.23.tgz",
- "integrity": "sha512-WHFH85DkCfiNMDX5D3X7hpNH3/PUhjTGcD0U1SgfBGZxJ3qUmJh5FdvaFjcClxOvB3rzdfj4oRffbI38jEnC1w==",
+ "version": "0.5.25",
+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.25.tgz",
+ "integrity": "sha512-DgEaTyN/z0HFaVcVbSyVCUU6HeFdnNC3vE4c9cgu2dgMTvjBUBdBzWfasTBmAW45u5OIMeCJtU8yNjM22DHucw==",
"requires": {
"moment": ">= 2.9.0"
}
diff --git a/shared/package.json b/shared/package.json
index a14d146e..f9a927b6 100644
--- a/shared/package.json
+++ b/shared/package.json
@@ -13,7 +13,7 @@
},
"dependencies": {
"moment": "^2.23.0",
- "moment-timezone": "^0.5.23",
+ "moment-timezone": "^0.5.25",
"owasp-password-strength-test": "github:bures/owasp-password-strength-test"
},
"devDependencies": {