From fcae5854174d4be585272d41a610dd1364bcd340 Mon Sep 17 00:00:00 2001 From: joker-x Date: Wed, 26 Aug 2020 06:54:18 +0200 Subject: [PATCH 1/4] Convert all tables in database to utf8mb4. Fix #928 --- server/config/default.yaml | 3 ++- server/models/lists.js | 10 +++++----- .../20200824160149_convert_to_utf8mb4.js | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 server/setup/knex/migrations/20200824160149_convert_to_utf8mb4.js diff --git a/server/config/default.yaml b/server/config/default.yaml index 14b07bf9..e23cc9c0 100644 --- a/server/config/default.yaml +++ b/server/config/default.yaml @@ -128,7 +128,8 @@ mysql: # The timezone configured on the MySQL server. This can be 'local', 'Z', or an offset in the form +HH:MM or -HH:MM # If the MySQL server runs on the same server as Mailtrain, use 'local' timezone: local - + multipleStatements: true + verp: # Enable to start an MX server that detects bounced messages using VERP # In most cases you do not want to use it diff --git a/server/models/lists.js b/server/models/lists.js index ada16cf8..84282977 100644 --- a/server/models/lists.js +++ b/server/models/lists.js @@ -207,13 +207,13 @@ async function create(context, entity) { await knex.schema.raw('CREATE TABLE `subscription__' + id + '` (\n' + ' `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' + - ' `cid` varchar(255) CHARACTER SET ascii NOT NULL,\n' + - ' `email` varchar(255) CHARACTER SET utf8 DEFAULT NULL,\n' + - ' `hash_email` varchar(255) CHARACTER SET ascii NOT NULL,\n' + + ' `cid` varchar(255) NOT NULL,\n' + + ' `email` varchar(255) DEFAULT NULL,\n' + + ' `hash_email` varchar(255) NOT NULL,\n' + ' `source_email` int(11) DEFAULT NULL,\n' + // Altough this is a reference to an import, it is represented as signed int(11). This is because we use negative values for constant from SubscriptionSource ' `opt_in_ip` varchar(100) DEFAULT NULL,\n' + ' `opt_in_country` varchar(2) DEFAULT NULL,\n' + - ' `tz` varchar(100) CHARACTER SET ascii DEFAULT NULL,\n' + + ' `tz` varchar(100) DEFAULT NULL,\n' + ' `status` tinyint(4) unsigned NOT NULL DEFAULT \'1\',\n' + ' `is_test` tinyint(4) unsigned NOT NULL DEFAULT \'0\',\n' + ' `status_change` timestamp NULL DEFAULT NULL,\n' + @@ -233,7 +233,7 @@ async function create(context, entity) { ' KEY `latest_click` (`latest_click`),\n' + ' KEY `created` (`created`),\n' + ' KEY `updated` (`updated`)\n' + - ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n'); + ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;\n'); await shares.rebuildPermissionsTx(tx, { entityTypeId: 'list', entityId: id }); diff --git a/server/setup/knex/migrations/20200824160149_convert_to_utf8mb4.js b/server/setup/knex/migrations/20200824160149_convert_to_utf8mb4.js new file mode 100644 index 00000000..e1a184f3 --- /dev/null +++ b/server/setup/knex/migrations/20200824160149_convert_to_utf8mb4.js @@ -0,0 +1,17 @@ + +exports.up = function(knex, Promise) { + return knex.raw('SELECT table_name FROM information_schema.tables WHERE table_schema = ?', [knex.client.database()]) + .then(function(tablas) { + let sql=""; + tablas=tablas[0]; + for(let i=0; i Date: Wed, 26 Aug 2020 08:00:12 +0200 Subject: [PATCH 2/4] Add utf8mb4 to new instalations from script --- setup/functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/functions b/setup/functions index fc691c28..db9e51ff 100644 --- a/setup/functions +++ b/setup/functions @@ -210,7 +210,7 @@ function installMailtrain { mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'localhost';" mysql -u root -e "CREATE USER 'mailtrain_ro'@'localhost' IDENTIFIED BY '$mysqlRoPassword';" mysql -u root -e "GRANT SELECT ON mailtrain.* TO 'mailtrain_ro'@'localhost';" - mysql -u mailtrain --password="$mysqlPassword" -e "CREATE database mailtrain;" + mysql -u mailtrain --password="$mysqlPassword" -e "CREATE DATABASE mailtrain CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;" # Add new user for the mailtrain daemon to run as useradd mailtrain || true @@ -233,6 +233,8 @@ www: mysql: password: "$mysqlPassword" + charset: utf8mb4 + multipleStatements: true redis: enabled: true From f610f93fa7cd10edd36e232512834c9c47cb8b04 Mon Sep 17 00:00:00 2001 From: joker-x Date: Wed, 26 Aug 2020 10:20:26 +0200 Subject: [PATCH 3/4] Remove charset and multipleStatements from config file, added hardcoded --- server/config/default.yaml | 2 -- server/lib/knex.js | 5 ++++- server/setup/knex/knexfile.js | 6 +++++- setup/functions | 2 -- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/server/config/default.yaml b/server/config/default.yaml index e23cc9c0..63769b29 100644 --- a/server/config/default.yaml +++ b/server/config/default.yaml @@ -124,11 +124,9 @@ mysql: # Some installations, eg. MAMP can use a different port (8889) # MAMP users should also turn on Allow network access to MySQL otherwise MySQL might not be accessible port: 3306 - charset: utf8mb4 # The timezone configured on the MySQL server. This can be 'local', 'Z', or an offset in the form +HH:MM or -HH:MM # If the MySQL server runs on the same server as Mailtrain, use 'local' timezone: local - multipleStatements: true verp: # Enable to start an MX server that detects bounced messages using VERP diff --git a/server/lib/knex.js b/server/lib/knex.js index 43cc7194..73d1b0ee 100644 --- a/server/lib/knex.js +++ b/server/lib/knex.js @@ -10,7 +10,10 @@ const knex = require('knex')({ connection: { ...config.mysql, - // DATE and DATETIME types contain no timezone info. The MySQL driver tries to interpret them w.r.t. to local time, which + charset: utf8mb4, + multipleStatements: true, + + // DATE and DATETIME types contain no timezone info. The MySQL driver tries to interpret them w.r.t. to local time, which // does not work well with assigning these values in UTC and handling them as if in UTC dateStrings: [ 'DATE', diff --git a/server/setup/knex/knexfile.js b/server/setup/knex/knexfile.js index ecb9416e..82622aea 100644 --- a/server/setup/knex/knexfile.js +++ b/server/setup/knex/knexfile.js @@ -1,6 +1,10 @@ 'use strict'; -const config = require('./config'); +let config = require('./config'); + +config.mysql.charset="utf8mb4"; +config.mysql.multipleStatements=true; + module.exports = { client: 'mysql', diff --git a/setup/functions b/setup/functions index db9e51ff..66cd8cc2 100644 --- a/setup/functions +++ b/setup/functions @@ -233,8 +233,6 @@ www: mysql: password: "$mysqlPassword" - charset: utf8mb4 - multipleStatements: true redis: enabled: true From 8f4b790b54717e5d2a1a0429e394b60b1822ddf1 Mon Sep 17 00:00:00 2001 From: joker-x Date: Wed, 26 Aug 2020 16:57:41 +0200 Subject: [PATCH 4/4] Fix syntax error --- server/lib/knex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/knex.js b/server/lib/knex.js index 73d1b0ee..266d6c0a 100644 --- a/server/lib/knex.js +++ b/server/lib/knex.js @@ -10,7 +10,7 @@ const knex = require('knex')({ connection: { ...config.mysql, - charset: utf8mb4, + charset: 'utf8mb4', multipleStatements: true, // DATE and DATETIME types contain no timezone info. The MySQL driver tries to interpret them w.r.t. to local time, which