From d3825eb4965981df1464e6b4c4d7de3968384975 Mon Sep 17 00:00:00 2001 From: Ylian Saint-Hilaire Date: Wed, 8 Jul 2020 11:59:20 -0700 Subject: [PATCH] Improved agent core dump collection system. --- db.js | 10 +++++----- meshagent.js | 30 ++++++++++++++++++++++++------ meshcentral-config-schema.json | 2 ++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/db.js b/db.js index 6809a7f0..90d9007c 100644 --- a/db.js +++ b/db.js @@ -669,7 +669,7 @@ module.exports.CreateDB = function (parent, func) { // Start NeDB main collection and setup indexes obj.file = new Datastore(datastoreOptions); - obj.file.persistence.setAutocompactionInterval(36000); + obj.file.persistence.setAutocompactionInterval(86400000); // Compact once a day obj.file.ensureIndex({ fieldName: 'type' }); obj.file.ensureIndex({ fieldName: 'domain' }); obj.file.ensureIndex({ fieldName: 'meshid', sparse: true }); @@ -678,14 +678,14 @@ module.exports.CreateDB = function (parent, func) { // Setup the events collection and setup indexes obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true }); - obj.eventsfile.persistence.setAutocompactionInterval(36000); + obj.eventsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field. obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true }); obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 20 }); // Limit the power event log to 20 days (Seconds * Minutes * Hours * Days) // Setup the power collection and setup indexes obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true }); - obj.powerfile.persistence.setAutocompactionInterval(36000); + obj.powerfile.persistence.setAutocompactionInterval(86400000); // Compact once a day obj.powerfile.ensureIndex({ fieldName: 'nodeid' }); obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 10 }); // Limit the power event log to 10 days (Seconds * Minutes * Hours * Days) @@ -694,14 +694,14 @@ module.exports.CreateDB = function (parent, func) { // Setup the server stats collection and setup indexes obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true }); - obj.serverstatsfile.persistence.setAutocompactionInterval(36000); + obj.serverstatsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: 60 * 60 * 24 * 30 }); // Limit the server stats log to 30 days (Seconds * Minutes * Hours * Days) obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events // Setup plugin info collection if (parent.config.settings != null) { obj.pluginsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-plugins.db'), autoload: true }); - obj.pluginsfile.persistence.setAutocompactionInterval(36000); + obj.pluginsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day } setupFunctions(func); // Completed setup of NeDB diff --git a/meshagent.js b/meshagent.js index 43fa539f..34a6b799 100644 --- a/meshagent.js +++ b/meshagent.js @@ -978,7 +978,10 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { // Set agent core dump if ((parent.parent.config.settings != null) && ((parent.parent.config.settings.agentcoredump === true) || (parent.parent.config.settings.agentcoredump === false))) { obj.send(JSON.stringify({ action: 'coredump', value: parent.parent.config.settings.agentcoredump })); - if (parent.parent.config.settings.agentcoredump === true) { obj.send(JSON.stringify({ action: 'getcoredump' })); } + if (parent.parent.config.settings.agentcoredump === true) { + // Check if we requested a core dump file in the last minute, if not, ask if one is present. + if ((parent.lastCoreDumpRequest == null) || ((Date.now() - parent.lastCoreDumpRequest) >= 60000)) { obj.send(JSON.stringify({ action: 'getcoredump' })); } + } } // Do this if IP location is enabled on this domain TODO: Set IP location per device group? @@ -1370,12 +1373,27 @@ module.exports.CreateMeshAgent = function (parent, db, ws, req, args, domain) { break; } case 'getcoredump': { + // Check if we requested a core dump file in the last minute, if so, ignore this. + if ((parent.lastCoreDumpRequest != null) && ((Date.now() - parent.lastCoreDumpRequest) < 60000)) break; + // Indicates if the agent has a coredump available - if (command.exists === true) { - //console.log('CoreDump for agent ' + obj.remoteaddrport); - obj.coreDumpPresent = true; - // TODO: We need to look at getting the dump uploaded to the server. - if (typeof command.agenthashhex == 'string') { obj.RequestCoreDump(command.agenthashhex); } + if ((command.exists === true) && (typeof command.agenthashhex == 'string') && (command.agenthashhex.length == 96)) { + // Check if we already have this exact dump file + const coreDumpFile = parent.path.join(parent.parent.datapath, 'coredumps', obj.agentInfo.agentId + '-' + command.agenthashhex + '-' + obj.nodeid + '.dmp'); + parent.fs.stat(coreDumpFile, function (err, stats) { + if (stats != null) return; + obj.coreDumpPresent = true; + + // Check how many files are in the coredumps folder + const coreDumpPath = parent.path.join(parent.parent.datapath, 'coredumps'); + parent.fs.readdir(coreDumpPath, function (err, files) { + if ((files != null) && (files.length >= 20)) return; // Don't get more than 20 core dump files. + + // Get the core dump uploaded to the server. + parent.lastCoreDumpRequest = Date.now(); + obj.RequestCoreDump(command.agenthashhex); + }); + }); } break; } diff --git a/meshcentral-config-schema.json b/meshcentral-config-schema.json index 4f09111c..e2e56098 100644 --- a/meshcentral-config-schema.json +++ b/meshcentral-config-schema.json @@ -39,6 +39,7 @@ "agentAliasPort": { "type": "integer", "minimum": 1, "maximum": 65535, "description": "When set, indicates the actual publically visible agent-only port. If not set, the AgentPort value is used." }, "agentAliasDNS": { "type": "string", "format": "hostname", "description": "When set, specified the DNS name used by agents to connect to the agent-only port." }, "agentPortTls": { "type": "boolean", "default": true, "description": "Indicates if the agent-only port must perform TLS, this should be set to false if TLS is performed in front of this server." }, + "agentCoreDump": { "type": "boolean", "default": false, "description": "Automatically activates and transfers any agent crash dump files to the server in meshcentral-data/coredumps." }, "exactPorts": { "type": "boolean", "default": false }, "allowLoginToken": { "type": "boolean", "default": false }, "allowFraming": { "type": "boolean", "default": false }, @@ -53,6 +54,7 @@ "agentPing": { "type": "integer", "minimum": 1, "description": "When specified, sends data to the agent at x seconds interval and expects a response from the agent." }, "agentPong": { "type": "integer", "minimum": 1, "description": "When specified, sends data to the agent at x seconds interval." }, "agentIdleTimeout": { "type": "integer", "minimum": 1 }, + "compression": { "type": "boolean", "default": true, "description": "Enables GZIP compression for web requests." }, "meshErrorLogPath": { "type": "string" }, "npmPath": { "type": "string" }, "npmProxy": { "type": "string", "format": "uri" },