mirror of
https://github.com/Ylianst/MeshCentral.git
synced 2025-03-09 15:40:18 +00:00
Add sqlite config options (#6517)
This commit is contained in:
PTR
GitHub
parent
b71c69e81d
commit
777eb53476
2 changed files with 125 additions and 22 deletions
111
db.js
111
db.js
|
|
@ -39,6 +39,17 @@ module.exports.CreateDB = function (parent, func) {
|
|||
let databaseName = 'meshcentral';
|
||||
let datapathParentPath = path.dirname(parent.datapath);
|
||||
let datapathFoldername = path.basename(parent.datapath);
|
||||
const SQLITE_AUTOVACUUM = ['none', 'full', 'incremental'];
|
||||
const SQLITE_SYNCHRONOUS = ['off', 'normal', 'full', 'extra'];
|
||||
obj.sqliteConfig = {
|
||||
maintenance: '',
|
||||
startupVacuum: false,
|
||||
autoVacuum: 'full',
|
||||
incrementalVacuum: 100,
|
||||
journalMode: 'delete',
|
||||
journalSize: 4096000,
|
||||
synchronous: 'full',
|
||||
};
|
||||
obj.performingBackup = false;
|
||||
const BACKUPFAIL_ZIPCREATE = 0x0001;
|
||||
const BACKUPFAIL_ZIPMODULE = 0x0010;
|
||||
|
|
@ -119,6 +130,7 @@ module.exports.CreateDB = function (parent, func) {
|
|||
|
||||
// Perform database maintenance
|
||||
obj.maintenance = function () {
|
||||
parent.debug('db', 'Entering database maintenance');
|
||||
if (obj.databaseType == DB_NEDB) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
|
||||
obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
|
||||
obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
|
||||
|
|
@ -138,12 +150,21 @@ module.exports.CreateDB = function (parent, func) {
|
|||
});
|
||||
});
|
||||
} else if (obj.databaseType == DB_SQLITE) { // SQLite3
|
||||
// TODO: Combine with others?
|
||||
sqlDbQuery('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))], function (doc, err) { });
|
||||
sqlDbQuery('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))], function (doc, err) { });
|
||||
sqlDbQuery('DELETE FROM serverstats WHERE expire < ?', [new Date()], function (doc, err) { });
|
||||
sqlDbQuery('DELETE FROM smbios WHERE expire < ?', [new Date()], function (doc, err) { });
|
||||
obj.file.run( 'PRAGMA optimize;' ); //see https://sqlite.org/pragma.html#pragma_optimize
|
||||
//sqlite does not return rows affected for INSERT, UPDATE or DELETE statements, see https://www.sqlite.org/pragma.html#pragma_count_changes
|
||||
obj.file.serialize(function () {
|
||||
obj.file.run('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))]);
|
||||
obj.file.run('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))]);
|
||||
obj.file.run('DELETE FROM serverstats WHERE expire < ?', [new Date()]);
|
||||
obj.file.run('DELETE FROM smbios WHERE expire < ?', [new Date()]);
|
||||
obj.file.exec(obj.sqliteConfig.maintenance, function (err) {
|
||||
if (err) {console.log('Maintenance error: ' + err.message)};
|
||||
if (parent.config.settings.debug) {
|
||||
sqliteGetPragmas(['freelist_count', 'page_size', 'page_count', 'cache_size' ], function (pragma, pragmaValue) {
|
||||
parent.debug('db', 'SQLite Maintenance: ' + pragma + '=' + pragmaValue);
|
||||
});
|
||||
};
|
||||
});
|
||||
});
|
||||
}
|
||||
obj.removeInactiveDevices();
|
||||
}
|
||||
|
|
@ -742,13 +763,29 @@ module.exports.CreateDB = function (parent, func) {
|
|||
// SQLite3 database setup
|
||||
obj.databaseType = DB_SQLITE;
|
||||
const sqlite3 = require('sqlite3');
|
||||
if (typeof parent.config.settings.sqlite3 == 'string') {databaseName = parent.config.settings.sqlite3};
|
||||
//use sqlite3 cache mode https://github.com/TryGhost/node-sqlite3/wiki/Caching#caching
|
||||
obj.file = new sqlite3.cached.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
|
||||
let configParams = parent.config.settings.sqlite3;
|
||||
if (typeof configParams == 'string') {databaseName = configParams} else {databaseName = configParams.name ? configParams.name : 'meshcentral';};
|
||||
obj.sqliteConfig.startupVacuum = configParams.startupvacuum ? configParams.startupvacuum : false;
|
||||
obj.sqliteConfig.autoVacuum = configParams.autovacuum ? configParams.autovacuum.toLowerCase() : 'incremental';
|
||||
obj.sqliteConfig.incrementalVacuum = configParams.incrementalvacuum ? configParams.incrementalvacuum : 100;
|
||||
obj.sqliteConfig.journalMode = configParams.journalmode ? configParams.journalmode.toLowerCase() : 'delete';
|
||||
//allowed modes, 'none' excluded because not usefull for this app, maybe also remove 'memory'?
|
||||
if (!(['delete', 'truncate', 'persist', 'memory', 'wal'].includes(obj.sqliteConfig.journalMode))) { obj.sqliteConfig.journalMode = 'delete'};
|
||||
obj.sqliteConfig.journalSize = configParams.journalsize ? configParams.journalsize : 409600;
|
||||
//wal can use the more performant 'normal' mode, see https://www.sqlite.org/pragma.html#pragma_synchronous
|
||||
obj.sqliteConfig.synchronous = (obj.sqliteConfig.journalMode == 'wal') ? 'normal' : 'full';
|
||||
if (obj.sqliteConfig.journalMode == 'wal') {obj.sqliteConfig.maintenance += 'PRAGMA wal_checkpoint(PASSIVE);'};
|
||||
if (obj.sqliteConfig.autoVacuum == 'incremental') {obj.sqliteConfig.maintenance += 'PRAGMA incremental_vacuum(' + obj.sqliteConfig.incrementalVacuum + ');'};
|
||||
obj.sqliteConfig.maintenance += 'PRAGMA optimize;';
|
||||
|
||||
parent.debug('db', 'SQlite config options: ' + JSON.stringify(obj.sqliteConfig, null, 4));
|
||||
if (obj.sqliteConfig.journalMode == 'memory') { console.log('[WARNING] journal_mode=memory: this can lead to database corruption if there is a crash during a transaction. See https://www.sqlite.org/pragma.html#pragma_journal_mode') };
|
||||
//.cached not usefull
|
||||
obj.file = new sqlite3.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
|
||||
if (err && (err.code == 'SQLITE_CANTOPEN')) {
|
||||
// Database needs to be created
|
||||
obj.file = new sqlite3.Database(parent.path.join(parent.datapath, databaseName + '.sqlite'), function (err) {
|
||||
if (err) { console.log("SQLite Error: " + err); process.exit(1);; return; }
|
||||
if (err) { console.log("SQLite Error: " + err); process.exit(1); }
|
||||
obj.file.exec(`
|
||||
CREATE TABLE main (id VARCHAR(256) PRIMARY KEY NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON);
|
||||
CREATE TABLE events(id INTEGER PRIMARY KEY, time TIMESTAMP, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON);
|
||||
|
|
@ -771,23 +808,18 @@ module.exports.CreateDB = function (parent, func) {
|
|||
CREATE INDEX ndxsmbiosexpire ON smbios (expire);
|
||||
`, function (err) {
|
||||
// Completed DB creation of SQLite3
|
||||
//WAL mode instead of roll-back/delete
|
||||
obj.file.run( 'PRAGMA journal_mode=WAL;' );
|
||||
//Together with the optimize in the maintenance run, see https://sqlite.org/pragma.html#pragma_optimize
|
||||
obj.file.run( 'PRAGMA optimize=0x10002;' );
|
||||
sqliteSetOptions(func);
|
||||
//setupFunctions could be put in the sqliteSetupOptions, but left after it for clarity
|
||||
setupFunctions(func);
|
||||
}
|
||||
);
|
||||
});
|
||||
return;
|
||||
} else if (err) { console.log("SQLite Error: " + err); process.exit(0); return; }
|
||||
} else if (err) { console.log("SQLite Error: " + err); process.exit(0); }
|
||||
|
||||
// Completed setup of SQLite3
|
||||
//for existing db's
|
||||
//WAL mode instead of roll-back/delete
|
||||
obj.file.run( 'PRAGMA journal_mode=WAL;' );
|
||||
//Together with the optimize in the maintenance run, see https://sqlite.org/pragma.html#pragma_optimize
|
||||
obj.file.run( 'PRAGMA optimize=0x10002;' );
|
||||
sqliteSetOptions();
|
||||
//setupFunctions could be put in the sqliteSetupOptions, but left after it for clarity
|
||||
setupFunctions(func);
|
||||
});
|
||||
} else if (parent.args.acebase) {
|
||||
|
|
@ -1277,6 +1309,45 @@ module.exports.CreateDB = function (parent, func) {
|
|||
setupFunctions(func); // Completed setup of NeDB
|
||||
}
|
||||
|
||||
function sqliteSetOptions(func) {
|
||||
//get current auto_vacuum mode for comparison
|
||||
obj.file.get('PRAGMA auto_vacuum;', function(err, current){
|
||||
let pragma = 'PRAGMA journal_mode=' + obj.sqliteConfig.journalMode + ';' +
|
||||
'PRAGMA synchronous='+ obj.sqliteConfig.synchronous + ';' +
|
||||
'PRAGMA journal_size_limit=' + obj.sqliteConfig.journalSize + ';' +
|
||||
'PRAGMA auto_vacuum=' + obj.sqliteConfig.autoVacuum + ';' +
|
||||
'PRAGMA incremental_vacuum=' + obj.sqliteConfig.incrementalVacuum + ';' +
|
||||
'PRAGMA optimize=0x10002;';
|
||||
//check new autovacuum mode, if changing from or to 'none', a VACUUM needs to be done to activate it. See https://www.sqlite.org/pragma.html#pragma_auto_vacuum
|
||||
if ( obj.sqliteConfig.startupVacuum
|
||||
|| (current.auto_vacuum == 0 && obj.sqliteConfig.autoVacuum !='none')
|
||||
|| (current.auto_vacuum != 0 && obj.sqliteConfig.autoVacuum =='none'))
|
||||
{
|
||||
pragma += 'VACUUM;';
|
||||
};
|
||||
parent.debug ('db', 'Config statement: ' + pragma);
|
||||
|
||||
obj.file.exec( pragma,
|
||||
function (err) {
|
||||
if (err) { parent.debug('db', 'Config pragma error: ' + (err.message)) };
|
||||
sqliteGetPragmas(['journal_mode', 'journal_size_limit', 'freelist_count', 'auto_vacuum', 'page_size', 'wal_autocheckpoint', 'synchronous'], function (pragma, pragmaValue) {
|
||||
parent.debug('db', 'PRAGMA: ' + pragma + '=' + pragmaValue);
|
||||
});
|
||||
});
|
||||
});
|
||||
//setupFunctions(func);
|
||||
}
|
||||
|
||||
function sqliteGetPragmas (pragmas, func){
|
||||
//pragmas can only be gotting one by one
|
||||
pragmas.forEach (function (pragma) {
|
||||
obj.file.get('PRAGMA ' + pragma + ';', function(err, res){
|
||||
if (pragma == 'auto_vacuum') { res[pragma] = SQLITE_AUTOVACUUM[res[pragma]] };
|
||||
if (pragma == 'synchronous') { res[pragma] = SQLITE_SYNCHRONOUS[res[pragma]] };
|
||||
if (func) { func (pragma, res[pragma]); }
|
||||
});
|
||||
});
|
||||
}
|
||||
// Create the PostgreSQL tables
|
||||
function postgreSqlCreateTables(func) {
|
||||
// Database was created, create the tables
|
||||
|
|
|
|||
|
|
@ -94,9 +94,41 @@
|
|||
}
|
||||
},
|
||||
"sqlite3": {
|
||||
"type": [ "boolean", "string" ],
|
||||
"type": [ "boolean", "string", "object" ],
|
||||
"default": false,
|
||||
"description": "Set boolean true to use SQLite3 as a local MeshCentral database with default db filename 'meshcentral' or enter a string for a different db filename. Extension .sqlite is appended"
|
||||
"description": "Set boolean true to use SQLite3 as a local MeshCentral database with default db filename 'meshcentral' or enter a string for a different db filename. Extension .sqlite is appended",
|
||||
"properties":{
|
||||
"name": {
|
||||
"type": "string",
|
||||
"default": "meshcentral",
|
||||
"description": "Database filename. '.sqlite' is appended"
|
||||
},
|
||||
"journalMode": {
|
||||
"type": "string",
|
||||
"default": "delete",
|
||||
"description": "DELETE, TRUNCATE, PERSIST, MEMORY, WAL. NONE not allowed. See: https://www.sqlite.org/pragma.html#pragma_journal_mode"
|
||||
},
|
||||
"journalSize": {
|
||||
"type": "integer",
|
||||
"default": 4096000,
|
||||
"description": "Maximum size of the journal file in bytes. Can grow larger if needed, but will shrink to this size. -1 is unlimited growth. See: https://www.sqlite.org/pragma.html#pragma_journal_size_limit"
|
||||
},
|
||||
"autoVacuum": {
|
||||
"type": "string",
|
||||
"default": "incremental",
|
||||
"description": "none, full, incremental. Removes unused pages and shrinks databasefile during maintenance. See: https://www.sqlite.org/pragma.html#pragma_auto_vacuum"
|
||||
},
|
||||
"incrementalVacuum": {
|
||||
"type": "integer",
|
||||
"default": 100,
|
||||
"description": "Maximum amount of pages to free during maintenance. Default page size is 4k, so default frees up to 400k from the databasefile. See: https://www.sqlite.org/pragma.html#pragma_incremental_vacuum"
|
||||
},
|
||||
"startupVacuum": {
|
||||
"type": "boolean",
|
||||
"default": false,
|
||||
"description": "Do a full VACUUM at startup. Shrinks the db file and optimizes it. This can take some time with a large database and can temporarily take up to double the database size on disk. See: https://www.sqlite.org/lang_vacuum.html"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mySQL": {
|
||||
"type": "object",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue