diff --git a/app.js b/app.js index 13268044..9d0f2a94 100644 --- a/app.js +++ b/app.js @@ -5,6 +5,8 @@ const log = require('npmlog'); const _ = require('./lib/translate')._; +const { nodeifyFunction } = require('./lib/nodeify'); + const express = require('express'); const bodyParser = require('body-parser'); const path = require('path'); @@ -23,8 +25,8 @@ const contextHelpers = require('./lib/context-helpers'); const routes = require('./routes/index'); const lists = require('./routes/lists-legacy'); -const settings = require('./routes/settings'); -const settingsModel = require('./lib/models/settings'); +//const settings = require('./routes/settings'); +const getSettings = nodeifyFunction(require('./models/settings').get); const templates = require('./routes/templates'); const campaigns = require('./routes/campaigns'); const links = require('./routes/links'); @@ -215,7 +217,7 @@ app.use((req, res, next) => { } res.locals.bodyClass = bodyClasses.join(' '); - settingsModel.list(['ua_code', 'shoutout'], (err, configItems) => { + getSettings(['uaCode', 'shoutout'], (err, configItems) => { if (err) { return next(err); } @@ -227,7 +229,7 @@ app.use((req, res, next) => { }); // Endpoint under /api are authenticated by access token -app.all('/api/*', passport.authByPanelToken); +app.all('/api/*', passport.authByAccessToken); // Marks the following endpoint to return JSON object when error occurs @@ -254,7 +256,7 @@ app.use('/', routes); app.use('/lists', lists); app.use('/templates', templates); app.use('/campaigns', campaigns); -app.use('/settings', settings); +//app.use('/settings', settings); app.use('/links', links); app.use('/fields', fields); app.use('/forms', forms); diff --git a/lib/dbcheck.js b/lib/dbcheck.js index aca1c644..8581ce9f 100644 --- a/lib/dbcheck.js +++ b/lib/dbcheck.js @@ -10,7 +10,8 @@ let log = require('npmlog'); let fs = require('fs'); let pathlib = require('path'); let Handlebars = require('handlebars'); -let meta = require('../meta.json'); + +const highestLegacySchemaVersion = 29; let mysqlConfig = { multipleStatements: true @@ -66,14 +67,27 @@ function getSchemaVersion(callback) { if (err) { return callback(err); } - connection.query('SELECT `value` FROM `settings` WHERE `key`=?', ['db_schema_version'], (err, rows) => { - connection.release(); - if (err) { - return callback(err); - } - let dbSchemaVersion = rows && rows[0] && Number(rows[0].value) || 0; - callback(null, dbSchemaVersion); + connection.query('SHOW TABLES LIKE "knex_migrations"', (err, rows) => { + if (rows) { + connection.release(); + + if (err) { + return callback(err); + } + + callback(null, highestLegacySchemaVersion); + } else { + connection.query('SELECT `value` FROM `settings` WHERE `key`=?', ['db_schema_version'], (err, rows) => { + connection.release(); + if (err) { + return callback(err); + } + + let dbSchemaVersion = rows && rows[0] && Number(rows[0].value) || 0; + callback(null, dbSchemaVersion); + }); + } }); }); } @@ -142,7 +156,7 @@ function runUpdates(callback, runCount) { return callback(err); } - if (schemaVersion >= meta.schemaVersion) { + if (schemaVersion >= highestLegacySchemaVersion) { // nothing to do here, already updated return callback(null, false); } diff --git a/lib/helpers.js b/lib/helpers.js index aee1af94..b525755d 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -1,18 +1,8 @@ 'use strict'; -let config = require('config'); -let path = require('path'); -let fs = require('fs'); -let tools = require('./tools'); -let settings = require('./models/settings'); let lists = require('./models/lists'); let fields = require('./models/fields'); -let forms = require('./models/forms'); let _ = require('./translate')._; -let objectHash = require('object-hash'); -let mjml = require('mjml'); -let mjmlTemplates = new Map(); -let hbs = require('hbs'); module.exports = { getDefaultMergeTags, diff --git a/lib/mailer.js b/lib/mailer.js index 4ef5ad0a..ee7f1baa 100644 --- a/lib/mailer.js +++ b/lib/mailer.js @@ -1,10 +1,12 @@ 'use strict'; +const { nodeifyFunction } = require('./nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let log = require('npmlog'); let config = require('config'); let nodemailer = require('nodemailer'); let openpgpEncrypt = require('nodemailer-openpgp').openpgpEncrypt; -let settings = require('./models/settings'); let tools = require('./tools'); let db = require('./db'); let Handlebars = require('handlebars'); @@ -169,7 +171,7 @@ function getTemplate(template, callback) { } function createMailer(callback) { - settings.list(['smtpHostname', 'smtpPort', 'smtpEncryption', 'smtpUser', 'smtpPass', 'smtpLog', 'smtpDisableAuth', 'smtpMaxConnections', 'smtpMaxMessages', 'smtpSelfSigned', 'pgpPrivateKey', 'pgpPassphrase', 'smtpThrottling', 'mailTransport', 'sesKey', 'sesSecret', 'sesRegion'], (err, configItems) => { + getSettings(['smtpHostname', 'smtpPort', 'smtpEncryption', 'smtpUser', 'smtpPass', 'smtpLog', 'smtpDisableAuth', 'smtpMaxConnections', 'smtpMaxMessages', 'smtpSelfSigned', 'pgpPrivateKey', 'pgpPassphrase', 'smtpThrottling', 'mailTransport', 'sesKey', 'sesSecret', 'sesRegion'], (err, configItems) => { if (err) { return callback(err); } diff --git a/lib/models/settings.js b/lib/models/settings.js deleted file mode 100644 index 9655ebd7..00000000 --- a/lib/models/settings.js +++ /dev/null @@ -1,77 +0,0 @@ -'use strict'; - -let tools = require('../tools'); -let db = require('../db'); - -module.exports = { - list: listValues, - get: getValue, - set: setValue -}; - -function listValues(filter, callback) { - if (!callback && typeof filter === 'function') { - callback = filter; - filter = false; - } - - // TODO: It would be good to cache the settings. It feels awkward to always go to DB to retrieve something what is essentially a constant - - filter = [].concat(filter || []).map(key => tools.toDbKey(key)); - - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - - let query; - - if (filter.length) { - query = 'SELECT * FROM settings WHERE `key` IN (' + filter.map(() => '?').join(',') + ')'; - } else { - query = 'SELECT * FROM settings'; - } - - connection.query(query, filter, (err, rows) => { - connection.release(); - if (err) { - return callback(err); - } - let settings = {}; - (rows || []).forEach(row => { - settings[row.key] = row.value; - }); - return callback(null, tools.convertKeys(settings)); - }); - }); -} - -function getValue(key, callback) { - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - connection.query('SELECT `value` FROM settings WHERE `key`=?', [tools.toDbKey(key)], (err, rows) => { - connection.release(); - if (err) { - return callback(err); - } - return callback(null, rows && rows[0] && rows[0].value || false); - }); - }); -} - -function setValue(key, value, callback) { - db.getConnection((err, connection) => { - if (err) { - return callback(err); - } - connection.query('INSERT INTO settings (`key`, `value`) VALUES (?,?) ON DUPLICATE KEY UPDATE `key`=?, `value`=?', [key, value, key, value], (err, response) => { - connection.release(); - if (err) { - return callback(err); - } - return callback(null, response && response.insertId || 0); - }); - }); -} diff --git a/meta.json b/meta.json deleted file mode 100644 index c0a6d3c0..00000000 --- a/meta.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "schemaVersion": 29 -} diff --git a/models/settings.js b/models/settings.js index 668ab93f..38e393ef 100644 --- a/models/settings.js +++ b/models/settings.js @@ -11,13 +11,11 @@ async function get(keyOrKeys) { keys = keyOrKeys; } - keys = keys.map(key => tools.toDbKey(key)); - const rows = await knex('settings').select(['key', 'value']).whereIn('key', keys); const settings = {}; for (const row of rows) { - settings[tools.fromDbKey(row.key)] = row.value; + settings[row.key] = row.value; } if (!Array.isArray(keyOrKeys)) { diff --git a/package.json b/package.json index 73422808..37774e79 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "node": ">=5.0.0" }, "devDependencies": { - "babel-eslint": "^8.0.0", + "babel-eslint": "^8.1.2", "bluebird": "^3.5.0", "chai": "^4.1.2", "eslint-config-nodemailer": "^1.2.0", @@ -48,13 +48,13 @@ }, "dependencies": { "async": "^2.5.0", - "aws-sdk": "^2.122.0", + "aws-sdk": "^2.176.0", "bcrypt-nodejs": "0.0.3", "bluebird": "^3.5.0", "body-parser": "^1.18.2", "bounce-handler": "^7.3.2-fork.2", "compression": "^1.7.0", - "config": "^1.26.2", + "config": "^1.29.0", "connect-flash": "^0.1.1", "connect-redis": "^3.3.0", "cookie-parser": "^1.4.3", @@ -68,11 +68,11 @@ "express": "^4.15.5", "express-session": "^1.15.5", "faker": "^4.1.0", - "feedparser": "^2.2.1", + "feedparser": "^2.2.7", "fs-extra": "^4.0.2", "geoip-ultralight": "^0.1.5", "gettext-parser": "^1.3.0", - "gm": "^1.23.0", + "gm": "^1.23.1", "handlebars": "^4.0.10", "hbs": "^4.0.1", "he": "^1.1.1", @@ -87,7 +87,7 @@ "knex": "^0.13.0", "libmime": "^3.1.0", "mailparser": "^2.0.5", - "marked": "^0.3.6", + "marked": "^0.3.9", "memory-cache": "^0.2.0", "mjml": "3.3.5", "mkdirp": "^0.5.1", @@ -104,7 +104,7 @@ "nodemailer-openpgp": "^1.1.0", "npmlog": "^4.1.2", "object-hash": "^1.1.8", - "openpgp": "^2.5.11", + "openpgp": "^2.6.1", "owasp-password-strength-test": "github:bures/owasp-password-strength-test", "passport": "^0.4.0", "passport-local": "^1.0.0", @@ -115,7 +115,7 @@ "request-promise": "^4.2.2", "serve-favicon": "^2.4.4", "shortid": "^2.2.8", - "slugify": "^1.2.1", + "slugify": "^1.2.8", "smtp-server": "^3.1.0", "striptags": "^3.1.0", "toml": "^2.3.3", diff --git a/routes/archive.js b/routes/archive.js index bd005db1..800b5162 100644 --- a/routes/archive.js +++ b/routes/archive.js @@ -1,6 +1,8 @@ 'use strict'; -let settings = require('../lib/models/settings'); +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let campaigns = require('../lib/models/campaigns'); let links = require('../lib/models/links'); let lists = require('../lib/models/lists'); @@ -15,7 +17,7 @@ let _ = require('../lib/translate')._; let util = require('util'); router.get('/:campaign/:list/:subscription', passport.csrfProtection, (req, res, next) => { - settings.get('serviceUrl', (err, serviceUrl) => { + getSettings('serviceUrl', (err, serviceUrl) => { if (err) { req.flash('danger', err.message || err); return res.redirect('/'); diff --git a/routes/campaigns.js b/routes/campaigns.js index 1bead6b3..b7fb9c50 100644 --- a/routes/campaigns.js +++ b/routes/campaigns.js @@ -1,5 +1,8 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let config = require('config'); let express = require('express'); let router = new express.Router(); @@ -7,7 +10,6 @@ let lists = require('../lib/models/lists'); let templates = require('../lib/models/templates'); let campaigns = require('../lib/models/campaigns'); let subscriptions = require('../lib/models/subscriptions'); -let settings = require('../lib/models/settings'); let tools = require('../lib/tools'); let editorHelpers = require('../lib/editor-helpers.js'); let striptags = require('striptags'); @@ -46,7 +48,7 @@ router.get('/create', passport.csrfProtection, (req, res) => { data.list = Number(data.list.split(':').shift()); } - settings.list(['defaultFrom', 'defaultAddress', 'defaultSubject'], (err, configItems) => { + getSettings(['defaultFrom', 'defaultAddress', 'defaultSubject'], (err, configItems) => { if (err) { req.flash('danger', err.message || err); return res.redirect('/'); @@ -138,7 +140,7 @@ router.get('/edit/:id', passport.csrfProtection, (req, res, next) => { } campaign.attachments = attachments; - settings.list(['disableWysiwyg'], (err, configItems) => { + getSettings(['disableWysiwyg'], (err, configItems) => { if (err) { return next(err); } diff --git a/routes/editorapi.js b/routes/editorapi.js index 58cbdf30..ca5e30f0 100644 --- a/routes/editorapi.js +++ b/routes/editorapi.js @@ -1,5 +1,8 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + const log = require('npmlog'); const config = require('config'); const express = require('express'); @@ -23,7 +26,6 @@ const htmlToText = require('html-to-text'); const premailerApi = require('premailer-api'); const _ = require('../lib/translate')._; const mailer = require('../lib/mailer'); -const settings = require('../lib/models/settings'); const templates = require('../lib/models/templates'); const campaigns = require('../lib/models/campaigns'); @@ -224,7 +226,7 @@ const getStaticImageUrl = (dynamicUrl, staticDir, staticDirUrl, callback) => { }; const prepareHtml = (html, editorName, callback) => { - settings.get('serviceUrl', (err, serviceUrl) => { + getSettings('serviceUrl', (err, serviceUrl) => { if (err) { return callback(err.message || err); } @@ -326,7 +328,7 @@ router.post('/update', passport.parseForm, passport.csrfProtection, (req, res) = // https://github.com/aguidrevitch/jquery-file-upload-middleware router.get('/upload', passport.csrfProtection, (req, res) => { - settings.get('serviceUrl', (err, serviceUrl) => { + getSettings('serviceUrl', (err, serviceUrl) => { if (err) { return res.status(500).send(err.message || err); } @@ -358,7 +360,7 @@ router.get('/upload', passport.csrfProtection, (req, res) => { }); router.post('/upload', passport.csrfProtection, (req, res) => { - settings.get('serviceUrl', (err, serviceUrl) => { + getSettings('serviceUrl', (err, serviceUrl) => { if (err) { return res.status(500).send(err.message || err); } @@ -474,7 +476,7 @@ router.post('/test', parseGrapejsMultipartTestForm, passport.csrfProtection, (re return sendError(err); } - settings.list(['defaultAddress', 'defaultFrom'], (err, configItems) => { + getSettings(['defaultAddress', 'defaultFrom'], (err, configItems) => { if (err) { return sendError(err); } diff --git a/routes/grapejs.js b/routes/grapejs.js index 780411fd..08962702 100644 --- a/routes/grapejs.js +++ b/routes/grapejs.js @@ -1,5 +1,8 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + const config = require('config'); const express = require('express'); const router = new express.Router(); @@ -7,9 +10,9 @@ const passport = require('../lib/passport'); const _ = require('../lib/translate')._; const fs = require('fs'); const path = require('path'); -const settings = require('../lib/models/settings'); const editorHelpers = require('../lib/editor-helpers') + router.all('/*', (req, res, next) => { if (!req.user) { req.flash('danger', _('Need to be logged in to access restricted content')); @@ -19,7 +22,7 @@ router.all('/*', (req, res, next) => { }); router.get('/editor', passport.csrfProtection, (req, res) => { - settings.get('serviceUrl', (err, serviceUrl) => { + getSettings('serviceUrl', (err, serviceUrl) => { if (err) { req.flash('danger', err.message || err); return res.redirect('/'); diff --git a/routes/links.js b/routes/links.js index 8242a618..537f1b52 100644 --- a/routes/links.js +++ b/routes/links.js @@ -1,7 +1,9 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let links = require('../lib/models/links'); -let settings = require('../lib/models/settings'); let lists = require('../lib/models/lists'); let subscriptions = require('../lib/models/subscriptions'); let tools = require('../lib/tools'); @@ -78,7 +80,7 @@ router.get('/:campaign/:list/:subscription/:link', (req, res) => { return notFound(); } - settings.get('serviceUrl', (err, serviceUrl) => { + getSettings('serviceUrl', (err, serviceUrl) => { if (err) { // ignore } diff --git a/routes/settings.js b/routes/settings.js index 32977b29..2a42c22a 100644 --- a/routes/settings.js +++ b/routes/settings.js @@ -1,5 +1,7 @@ 'use strict'; +// FIXME: This does not work now because of missing lib/models/settings + let config = require('config'); let passport = require('../lib/passport'); let express = require('express'); diff --git a/routes/subscription.js b/routes/subscription.js index 6a68a5d8..a49766dd 100644 --- a/routes/subscription.js +++ b/routes/subscription.js @@ -95,7 +95,7 @@ async function injectCustomFormData(customFormId, viewKey, data) { data.template.layout = form.layout || data.template.layout; data.formInputStyle = form.formInputStyle || '@import url(/subscription/form-input-style.css);'; - const configItems = await settings.get(['ua_code']); + const configItems = await settings.get(['uaCode']); data.uaCode = configItems.uaCode; data.customSubscriptionScripts = config.customsubscriptionscripts || []; @@ -243,7 +243,7 @@ router.getAsync('/:cid/widget', cors(corsOptions), async (req, res) => { const list = await lists.getByCid(req.params.cid); - const configItems = settings.get(['serviceUrl', 'pgpPrivateKey']); + const configItems = await settings.get(['serviceUrl', 'pgpPrivateKey']); const data = { title: list.name, diff --git a/routes/templates.js b/routes/templates.js index 0b503897..2382a7f8 100644 --- a/routes/templates.js +++ b/routes/templates.js @@ -1,10 +1,12 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let config = require('config'); let express = require('express'); let router = new express.Router(); let templates = require('../lib/models/templates'); -let settings = require('../lib/models/settings'); let tools = require('../lib/tools'); let helpers = require('../lib/helpers'); let striptags = require('striptags'); @@ -36,7 +38,7 @@ router.get('/create', passport.csrfProtection, (req, res, next) => { data.csrfToken = req.csrfToken(); data.useEditor = true; - settings.list(['defaultPostaddress', 'defaultSender', 'disableWysiwyg'], (err, configItems) => { + getSettings(['defaultPostaddress', 'defaultSender', 'disableWysiwyg'], (err, configItems) => { if (err) { return next(err); } @@ -95,7 +97,7 @@ router.get('/edit/:id', passport.csrfProtection, (req, res, next) => { req.flash('danger', err && err.message || err || _('Could not find template with specified ID')); return res.redirect('/templates'); } - settings.list(['disableWysiwyg'], (err, configItems) => { + getSettings(['disableWysiwyg'], (err, configItems) => { if (err) { return next(err); } diff --git a/routes/webhooks.js b/routes/webhooks.js index 957c3cb7..af584caf 100644 --- a/routes/webhooks.js +++ b/routes/webhooks.js @@ -1,10 +1,12 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let express = require('express'); let router = new express.Router(); let request = require('request'); let campaigns = require('../lib/models/campaigns'); -let settings = require('../lib/models/settings'); let log = require('npmlog'); let multer = require('multer'); let uploads = multer(); @@ -293,7 +295,7 @@ router.post('/zone-mta/sender-config', (req, res) => { error: 'api_token value not set' }); } - settings.list(['dkim_api_key', 'dkim_private_key', 'dkim_selector', 'dkim_domain'], (err, configItems) => { + getSettings(['dkim_api_key', 'dkim_private_key', 'dkim_selector', 'dkim_domain'], (err, configItems) => { if (err) { return res.json({ error: err.message diff --git a/services/sender.js b/services/sender.js index a30b38ee..415b8eb5 100644 --- a/services/sender.js +++ b/services/sender.js @@ -1,5 +1,8 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let log = require('npmlog'); let config = require('config'); let db = require('../lib/db'); @@ -10,7 +13,6 @@ let segments = require('../lib/models/segments'); let lists = require('../lib/models/lists'); let blacklist = require('../lib/models/blacklist'); let fields = require('../lib/models/fields'); -let settings = require('../lib/models/settings'); let links = require('../lib/models/links'); let shortid = require('shortid'); let url = require('url'); @@ -312,7 +314,7 @@ function formatMessage(message, callback) { return callback(new Error(_('List not found'))); } - settings.list(['serviceUrl', 'verpUse', 'verpHostname'], (err, configItems) => { + getSettings(['serviceUrl', 'verpUse', 'verpHostname'], (err, configItems) => { if (err) { return callback(err); } diff --git a/services/verp-server.js b/services/verp-server.js index 24150147..8c738e82 100644 --- a/services/verp-server.js +++ b/services/verp-server.js @@ -1,8 +1,10 @@ 'use strict'; +const { nodeifyFunction } = require('../lib/nodeify'); +const getSettings = nodeifyFunction(require('../models/settings').get); + let log = require('npmlog'); let config = require('config'); -let settings = require('../lib/models/settings'); let campaigns = require('../lib/models/campaigns'); let BounceHandler = require('bounce-handler').BounceHandler; let SMTPServer = require('smtp-server').SMTPServer; @@ -19,7 +21,7 @@ let server = new SMTPServer({ onRcptTo: (address, session, callback) => { - settings.list(['verpHostname'], (err, configItems) => { + getSettings(['verpHostname'], (err, configItems) => { if (err) { err = new Error('Failed to load configuration'); err.responseCode = 421; diff --git a/setup/knex/knexfile.js b/setup/knex/knexfile.js index 3611bea4..1e9ec995 100644 --- a/setup/knex/knexfile.js +++ b/setup/knex/knexfile.js @@ -3,6 +3,6 @@ const config = require('./config'); module.exports = { - client: 'mysql', + client: 'mysql2', connection: config.mysql }; diff --git a/setup/knex/migrations/20171230134232_convert_settings_to_camel_case.js b/setup/knex/migrations/20171230134232_convert_settings_to_camel_case.js new file mode 100644 index 00000000..a19aaa42 --- /dev/null +++ b/setup/knex/migrations/20171230134232_convert_settings_to_camel_case.js @@ -0,0 +1,24 @@ +"use strict"; + +function fromDbKey(key) { + let prefix = ''; + if (key.startsWith('_')) { + key = key.substring(1); + prefix = '_'; + + } + return prefix + key.replace(/[_-]([a-z])/g, (m, c) => c.toUpperCase()); +} + +exports.up = (knex, Promise) => (async() => { + const rows = await knex('settings'); + + for (const row of rows) { + await knex('settings').where('id', row.id).update('key', fromDbKey(row.key)) + } + +})(); + + +exports.down = (knex, Promise) => (async() => { +})(); \ No newline at end of file diff --git a/setup/knex/migrations/20171230172244_delete_schema_version.js b/setup/knex/migrations/20171230172244_delete_schema_version.js new file mode 100644 index 00000000..876afe0c --- /dev/null +++ b/setup/knex/migrations/20171230172244_delete_schema_version.js @@ -0,0 +1,9 @@ +"use strict"; + +exports.up = (knex, Promise) => (async() => { + await knex('settings').where('key', 'dbSchemaVersion').del(); +})(); + + +exports.down = (knex, Promise) => (async() => { +})(); \ No newline at end of file