external/MeshCentral
1
0
Fork 0
mirror of https://github.com/Ylianst/MeshCentral.git synced 2025-02-12 11:01:52 +00:00
Code Issues Releases Wiki Activity

MongoDB speed boost using bulkWrite()

Browse source
This commit is contained in:
Ylian Saint-Hilaire 2021-01-10 04:53:30 -08:00
parent 3249d727b8
commit e41ac08221
2 changed files with 28 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
db.js
View file

@ -38,6 +38,8 @@ module.exports.CreateDB = function (parent, func) {
obj.dbRecordsDecryptKey = null; obj.dbRecordsDecryptKey = null;
obj.changeStream = false; obj.changeStream = false;
obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false))); obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
obj.pendingSet = false;
obj.pendingSets = null;
obj.SetupDatabase = function (func) { obj.SetupDatabase = function (func) {
// Check if the database unique identifier is present // Check if the database unique identifier is present
@ -1029,7 +1031,17 @@ module.exports.CreateDB = function (parent, func) {
} }
} else if (obj.databaseType == 3) { } else if (obj.databaseType == 3) {
// Database actions on the main collection (MongoDB) // Database actions on the main collection (MongoDB)
obj.Set = function (data, func) { data = common.escapeLinksFieldNameEx(data); obj.file.replaceOne({ _id: data._id }, performTypedRecordEncrypt(data), { upsert: true }, func); }; obj.Set = function (data) { // Fast Set operation using bulkWrite(), this is much faster then using replaceOne()
if (obj.pendingSet == false) {
// Perform the operation now
obj.pendingSet = true; obj.pendingSets = null;
obj.file.bulkWrite([{ replaceOne: { filter: { _id: data._id }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(data)), upsert: true } }], bulkWriteCompleted);
} else {
// Add this operation to the pending list
if (obj.pendingSets == null) { obj.pendingSets = {} }
obj.pendingSets[data._id] = data;
}
};
obj.Get = function (id, func) { obj.Get = function (id, func) {
if (arguments.length > 2) { if (arguments.length > 2) {
var parms = [func]; var parms = [func];
@ -1408,6 +1420,20 @@ module.exports.CreateDB = function (parent, func) {
} }
} }
// MongoDB pending bulk write operation, perform fast bulk document replacement.
function bulkWriteCompleted() {
if (obj.pendingSets != null) {
// Perform pending operations
var ops = [], c = 0;
for (var i in obj.pendingSets) { c++; ops.push({ replaceOne: { filter: { _id: i }, replacement: performTypedRecordEncrypt(common.escapeLinksFieldNameEx(obj.pendingSets[i])), upsert: true } }); }
obj.file.bulkWrite(ops, bulkWriteCompleted);
obj.pendingSets = null;
} else {
// All done, no pending operations.
obj.pendingSet = false;
}
}
// Perform a server backup // Perform a server backup
obj.performingBackup = false; obj.performingBackup = false;
obj.performBackup = function (func) { obj.performBackup = function (func) {

2
webserver.js
View file

@ -4972,7 +4972,7 @@ module.exports.CreateWebServer = function (parent, db, args, certificates) {
// Add HTTP security headers to all responses // Add HTTP security headers to all responses
obj.app.use(function (req, res, next) { obj.app.use(function (req, res, next) {
// Useful for debugging reverse proxy issues // Useful for debugging reverse proxy issues
parent.debug('httpheaders', req.headers); parent.debug('httpheaders', req.method, req.url, req.headers);
// Set the real IP address of the request // Set the real IP address of the request
// If a trusted reverse-proxy is sending us the remote IP address, use it. // If a trusted reverse-proxy is sending us the remote IP address, use it.