From 97bd723e4e47af6a3f16802ee5a81746434d1121 Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Tue, 2 Nov 2021 19:44:41 -0700 Subject: [PATCH] Early work on Postgres SQL support. --- MeshCentralServer.njsproj | 8 ++++++ db.js | 53 +++++++++++++++++++++++++++++++++------ meshcentral.js | 1 + package.json | 18 +++++++++++-- 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/MeshCentralServer.njsproj b/MeshCentralServer.njsproj index 82b29408..bc6e6a22 100644 --- a/MeshCentralServer.njsproj +++ b/MeshCentralServer.njsproj @@ -645,6 +645,7 @@ + @@ -655,10 +656,13 @@ + + + @@ -681,6 +685,7 @@ + @@ -691,10 +696,13 @@ + + + diff --git a/db.js b/db.js index 0496de7a..4a6b3618 100644 --- a/db.js +++ b/db.js @@ -230,7 +230,7 @@ module.exports.CreateDB = function (parent, func) { // TODO: Remove all meshes that dont have any links // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now. - if ((obj.databaseType == 4) || (obj.databaseType == 5)) { + if ((obj.databaseType == 4) || (obj.databaseType == 5) || (obj.databaseType == 6)) { // MariaDB or MySQL obj.RemoveAllOfType('event', function () { }); obj.RemoveAllOfType('power', function () { }); @@ -640,6 +640,43 @@ module.exports.CreateDB = function (parent, func) { }); setTimeout(function () { tempDatastore.end(); }, 2000); } + } else if (parent.args.postgres) { + // Postgres SQL + var connectinArgs = parent.args.postgres; + var dbname = (connectinArgs.database != null) ? connectinArgs.database : 'meshcentral'; + obj.databaseType = 6; + const pgtools = require('pgtools'); + pgtools.createdb(connectinArgs, dbname, function (err, res) { + const { Pool, Client } = require('pg'); + connectinArgs.database = dbname; + Datastore = new Client(connectinArgs); + Datastore.connect() + parent.debug('db', 'Checking tables...'); + sqlDbBatchExec([ + 'CREATE TABLE IF NOT EXISTS main (id VARCHAR(256) PRIMARY KEY NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON)', + 'CREATE TABLE IF NOT EXISTS events(id SERIAL PRIMARY KEY, time TIMESTAMP, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON)', + 'CREATE TABLE IF NOT EXISTS eventids(fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT)', + 'CREATE TABLE IF NOT EXISTS serverstats (time TIMESTAMP PRIMARY KEY, expire TIMESTAMP, doc JSON)', + 'CREATE TABLE IF NOT EXISTS power (id SERIAL PRIMARY KEY, time TIMESTAMP, nodeid CHAR(255), doc JSON)', + 'CREATE TABLE IF NOT EXISTS smbios (id CHAR(255) PRIMARY KEY, time TIMESTAMP, expire TIMESTAMP, doc JSON)', + 'CREATE TABLE IF NOT EXISTS plugin (id SERIAL PRIMARY KEY, doc JSON)' + ], function (results) { + parent.debug('db', 'Checking indexes...'); + sqlDbExec('CREATE INDEX ndxtypedomainextra ON main (type, domain, extra)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxextra ON main (extra)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxextraex ON main (extraex)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxeventstime ON events(time)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxeventsusername ON events(domain, userid, time)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxeventsdomainnodeidtime ON events(domain, nodeid, time)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxeventids ON eventids(target)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxserverstattime ON serverstats (time)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxserverstatexpire ON serverstats (expire)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxpowernodeidtime ON power (nodeid, time)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxsmbiostime ON smbios (time)', null, function (err, response) { }); + sqlDbExec('CREATE INDEX ndxsmbiosexpire ON smbios (expire)', null, function (err, response) { }); + setupFunctions(func); + }); + }); } else if (parent.args.mongodb) { // Use MongoDB obj.databaseType = 3; @@ -1011,12 +1048,12 @@ module.exports.CreateDB = function (parent, func) { .then(function (rows) { conn.release(); const docs = []; - for (var i in rows) { if (rows[i].doc) { docs.push(performTypedRecordDecrypt((typeof rows[i].doc == 'object')? rows[i].doc : JSON.parse(rows[i].doc))); } } + for (var i in rows) { if (rows[i].doc) { docs.push(performTypedRecordDecrypt((typeof rows[i].doc == 'object') ? rows[i].doc : JSON.parse(rows[i].doc))); } } if (func) try { func(null, docs); } catch (ex) { console.log('SQLERR1', ex); } }) .catch(function (err) { conn.release(); if (func) try { func(err); } catch (ex) { console.log('SQLERR2', ex); } }); }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log('SQLERR3', ex); } } }); - } else if (obj.databaseType == 5) { // MySQL + } else if ((obj.databaseType == 5) || (obj.databaseType == 6)) { // MySQL or Postgres SQL Datastore.query(query, args, function (error, results, fields) { if (error != null) { if (func) try { func(error); } catch (ex) { console.log('SQLERR4', ex); } @@ -1041,10 +1078,10 @@ module.exports.CreateDB = function (parent, func) { if (func) try { func(null, rows[0]); } catch (ex) { console.log(ex); } }) .catch(function (err) { conn.release(); if (func) try { func(err); } catch (ex) { console.log(ex); } }); - }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log(ex); } } }); - } else if (obj.databaseType == 5) { // MySQL + }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log(ex); } } }); + } else if ((obj.databaseType == 5) || (obj.databaseType == 6)) { // MySQL or Postgres SQL Datastore.query(query, args, function (error, results, fields) { - if (func) try { func(error, results?results[0]:null); } catch (ex) { console.log(ex); } + if (func) try { func(error, results ? results[0] : null); } catch (ex) { console.log(ex); } }); } } @@ -1061,7 +1098,7 @@ module.exports.CreateDB = function (parent, func) { .catch(function (err) { conn.release(); if (func) { try { func(err); } catch (ex) { console.log(ex); } } }); }) .catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log(ex); } } }); - } else if (obj.databaseType == 5) { // MySQL + } else if ((obj.databaseType == 5) || (obj.databaseType == 6)) { // MySQL or Postgres SQL var Promises = []; for (var i in queries) { if (typeof queries[i] == 'string') { Promises.push(Datastore.query(queries[i])); } else { Promises.push(Datastore.query(queries[i][0], queries[i][1])); } } Promise.all(Promises) @@ -1071,7 +1108,7 @@ module.exports.CreateDB = function (parent, func) { } function setupFunctions(func) { - if ((obj.databaseType == 4) || (obj.databaseType == 5)) { + if ((obj.databaseType == 4) || (obj.databaseType == 5) || (obj.databaseType == 6)) { // Database actions on the main collection (MariaDB or MySQL) obj.Set = function (value, func) { obj.dbCounters.fileSet++; diff --git a/meshcentral.js b/meshcentral.js index 2ae7524a..d61fa0a7 100644 --- a/meshcentral.js +++ b/meshcentral.js @@ -3341,6 +3341,7 @@ function mainStart() { if (config.settings.mysql != null) { modules.push('mysql'); } // Add MySQL. //if (config.settings.mysql != null) { modules.push('@mysql/xdevapi'); } // Add MySQL, official driver (https://dev.mysql.com/doc/dev/connector-nodejs/8.0/) if (config.settings.mongodb != null) { modules.push('mongodb@4.1.0'); modules.push('saslprep'); } // Add MongoDB, official driver. + if (config.settings.postgres != null) { modules.push('pg@8.7.1'); modules.push('pgtools@0.3.2'); } // Add Postgres, Postgres driver. if (config.settings.mariadb != null) { modules.push('mariadb'); } // Add MariaDB, official driver. if (config.settings.vault != null) { modules.push('node-vault'); } // Add official HashiCorp's Vault module. if (config.settings.plugins != null) { modules.push('semver'); } // Required for version compat testing and update checks diff --git a/package.json b/package.json index c6187662..756dc7d1 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,9 @@ "sample-config-advanced.json" ], "dependencies": { + "@yetzt/nedb": "^1.8.0", + "archiver": "^4.0.2", + "archiver-zip-encrypted": "^1.0.10", "body-parser": "^1.19.0", "cbor": "~5.2.0", "compression": "^1.7.4", @@ -43,13 +46,24 @@ "express": "^4.17.0", "express-handlebars": "^3.1.0", "express-ws": "^4.0.0", + "image-size": "^1.0.0", "ipcheck": "^0.1.0", + "loadavg-windows": "^1.1.1", "minimist": "^1.2.5", + "mongodb": "^4.1.0", "multiparty": "^4.2.1", - "@yetzt/nedb": "^1.8.0", "node-forge": "^0.10.0", + "node-rdpjs-2": "^0.3.5", + "node-windows": "^0.1.4", + "otplib": "^10.2.3", + "pg": "^8.7.1", + "pgtools": "^0.3.2", + "saslprep": "^1.0.3", + "ssh2": "^1.5.0", + "web-push": "^3.4.5", "ws": "^5.2.3", - "yauzl": "^2.10.0" + "yauzl": "^2.10.0", + "yubikeyotp": "^0.2.0" }, "repository": { "type": "git",