From ed48b3227a5428f03b2b79d79ece5ec1db6b9b69 Mon Sep 17 00:00:00 2001 From: Tlams Date: Fri, 27 Apr 2018 16:24:31 +0100 Subject: [PATCH] fix logger issues --- code/scripts/main/api/v1/api.py | 2 +- code/scripts/main/core/core.py | 65 +++++++++++++++++++++++---- code/scripts/main/private/conf/config | 7 +-- code/scripts/main/startup.py | 38 ++++++++++------ 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/code/scripts/main/api/v1/api.py b/code/scripts/main/api/v1/api.py index 2af8beb..d868028 100644 --- a/code/scripts/main/api/v1/api.py +++ b/code/scripts/main/api/v1/api.py @@ -217,7 +217,7 @@ class Instance: class ThreadAPI(threading.Thread): #def __init__(self, threadid, name, urls, c, g, r): - def __init__(self, threadid, name, urls, c, g): + def __init__(self, threadid, name, urls, c, g, logger): """ Pass Global var in this theard.""" global core, generalconf, redis_msg core = c diff --git a/code/scripts/main/core/core.py b/code/scripts/main/core/core.py index f3f9ed1..deabc99 100644 --- a/code/scripts/main/core/core.py +++ b/code/scripts/main/core/core.py @@ -18,28 +18,32 @@ import time import base64 import hashlib - -def RunAnalyse(clusters_conf, generalconf, delay=300): - play = Analyse(clusters_conf, generalconf) - +def RunAnalyse(clusters_conf, generalconf, logger): + play = Analyse(clusters_conf, generalconf, logger) while True: """ Instances types availables: lxc/qemu/all""" play.run("all") - time.sleep(int(delay)) + time.sleep(int(generalconf["analyst"]["walker"])) class Core: - # def __init__(self, generalconf, Lredis): - def __init__(self, generalconf): + def __init__(self, generalconf, logger): self.generalconf = generalconf + self.logger = logger + self.logger.write({"thread":threading.get_ident() ,"result": "INFO", "type": "HYPERPROXMOX", + "value": "Start Core process"}) """ LOAD MONGODB """ + self.logger.write({"thread": threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", + "value": "MongoDB connection"}) self.mongo = MongoDB(generalconf["mongodb"]["ip"]) self.mongo.client = self.mongo.connect() """ LOAD REDIS """ + self.logger.write({"thread": threading.get_ident(),"result": "INFO", "type": "HYPERPROXMOX", + "value": "Redis connection"}) self.redis_msg = Redis_wrapper(generalconf["redis"]["ip"], generalconf["redis"]["port"], 0) @@ -62,13 +66,14 @@ class Core: self.clusters_conf = self.mongo.get_clusters_conf()["value"] """ Clean previous lockers """ + self.logger.write({"thread": threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", + "value": "Clean Locker"}) locker = Locker() locker.unlock(generalconf["analyst"]["walker_lock"], "startup") thc = threading.Thread(name="Update statistics", target=RunAnalyse, - args=(self.clusters_conf, self.generalconf, - generalconf["analyst"]["walker"])) + args=(self.clusters_conf, self.generalconf, self.logger)) thc.start() else: exit(1) @@ -454,6 +459,48 @@ class Core: clusters_delete = self.mongo.delete_clusters_conf(cluster) return clusters_delete + """ + ####################### + # DATA MANAGEMENT # + ####################### + """ + + def managedata(self, json): + if json["action"] == "purge": + if json["type"] == "strict": + listdate = self.mongo.get_all_datekey() + if json["date"] in listdate: + for date in listdate: + if date <= json["date"]: + for instance in self.mongo.get_instances(date): + self.mongo.deletedata("instances", instance["_id"]) + for node in self.mongo.get_nodes(date): + self.mongo.deletedata("nodes", node["_id"]) + for storage in self.mongo.get_storages(date): + self.mongo.deletedata("storages", storage["_id"]) + for disk in self.mongo.get_disks(date): + self.mongo.deletedata("disks", disk["_id"]) + + else: + purge = { + "value": "This date is not available", + "result": "WARNING", + "type": "HYPERPROXMOX" + } + + elif json["type"] == "sequencial": + purge = { + "value": "Not implemented", + "result": "WARNING", + "type": "HYPERPROXMOX" + } + else: + purge = { + "value": "Unknown purging type", + "result": "WARNING", + "type": "HYPERPROXMOX" + } + return purge """ ####################### diff --git a/code/scripts/main/private/conf/config b/code/scripts/main/private/conf/config index efbea8b..29c1c3a 100644 --- a/code/scripts/main/private/conf/config +++ b/code/scripts/main/private/conf/config @@ -46,11 +46,8 @@ walker: 300 walker_lock: /tmp/hyperproxmoxwalker.lock [logger] -; Active or not(True/False - Case sensitive !) -debug = False - -; debug level 1: "INFO", 2: "WARNING", 3: "ERROR", 4: "CRITICAL", 5: "DEBUG" -logs_level = 1 +; logs level 1: "INFO", 2: "WARNING", 3: "ERROR", 4: "CRITICAL", 5: "DEBUG" +logs_level = 5 ; Limit IO write, if debug level is active, this value is overwrite to 0 bulk_write = 1 diff --git a/code/scripts/main/startup.py b/code/scripts/main/startup.py index 73a763e..f1b476e 100644 --- a/code/scripts/main/startup.py +++ b/code/scripts/main/startup.py @@ -16,17 +16,24 @@ import getpass import os import stat import urllib3 -global passhash +import argparse + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) if __name__ == "__main__": + """ Arg parse""" + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument("-a", "--api", action="store_true", help="Start only") + args = parser.parse_args() + """ Read conf """ localconf = configparser.ConfigParser() localconf.read('private/conf/config') generalconf = { - "logger": {"debug": localconf['logger']['debug'], "logs_level": localconf['logger']['logs_level'], + "logger": {"logs_level": localconf['logger']['logs_level'], "logs_dir": localconf['logger']['logs_dir'], "bulk_write": localconf['logger']['bulk_write'], "bulk_size": localconf['logger']['bulk_size']}, @@ -42,8 +49,8 @@ if __name__ == "__main__": """ Active logger""" logger = Logger(generalconf["logger"]) - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": "Start logger"}) - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": ">>>>>>> -- NEW STARTUP -- <<<<<<<"}) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": "Start logger"}) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": ">>>>>>> -- NEW STARTUP -- <<<<<<<"}) CritConf = CryticalData() """ Step One: test private key or create it """ @@ -56,7 +63,7 @@ if __name__ == "__main__": print("This action can take some minutes, please wait.") gen = CritConf.generate_key(localconf['system']['key_pvt'], localconf['system']['key_pub'], passhash) if gen['result'] == "OK": - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": "Key generated in {0}".format(localconf['system']['key_pvt'])}) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": "Key generated in {0}".format(localconf['system']['key_pvt'])}) print("Your new key has been generate ! " "\n - Private Key: {0} " "\n - Public Key: {1}" @@ -66,7 +73,7 @@ if __name__ == "__main__": key_pvt = CritConf.read_private_key(localconf['system']['key_pvt'], passhash) else: print(gen['Error']) - logger.write({"result": "ERROR", "type": "HYPERPROXMOX", "value": "Your key is not create due to an error: {0}".format(gen['value'])}) + logger.write({"thread":threading.get_ident(), "result": "ERROR", "type": "HYPERPROXMOX", "value": "Your key is not create due to an error: {0}".format(gen['value'])}) exit(1) """ Test valid right for your private Key """ @@ -76,7 +83,7 @@ if __name__ == "__main__": format(oct(stat.S_IMODE(os.stat(localconf['system']['key_pvt']).st_mode)))) os.chmod(localconf['system']['key_pvt'], 0o600) print("Auto correction... done !") - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": "Setting chmod on your key.."}) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": "Setting chmod on your key.."}) """ Step two""" if 'passhash' not in vars(): @@ -85,10 +92,10 @@ if __name__ == "__main__": if key_pvt['result'] != "OK": print("{0}: {1} " "\nPlease verify your passphrase".format(key_pvt['type'], key_pvt['value'])) - logger.write({"result": "WARNING", "type": "HYPERPROXMOX", "value": "Bad passphrase, try again."}) + logger.write({"thread":threading.get_ident(), "result": "WARNING", "type": "HYPERPROXMOX", "value": "Bad passphrase, try again."}) exit(1) - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": "Loading keys in memory"}) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": "Loading keys in memory"}) key_pub = CritConf.read_public_key(localconf['system']['key_pub']) generalconf["keys"] = {"key_pvt": key_pvt["value"], "key_pub": key_pub["value"]} @@ -103,13 +110,16 @@ if __name__ == "__main__": '/api/v1/instance/id/([a-z0-9]+)/status/(start|stop|current|reset|shutdown)', 'Instance', # AUTH - '/api/v1/login', 'Login' + # '/api/v1/login', 'Login' # MANAGEMENT CLUSTER '/api/v1/administration/cluster/(?:[0-9a-zA-Z\_\-]+)', 'Cluster', '/api/v1/administration/cluster/', 'Cluster', # '/api/v1/administration/cluster/new', 'Cluster', + # PURGE SYSTEM + '/api/v1/administration/purge/([a-z0-9]+)', 'Purge', + # CACHE DATA (MONGO) # date/cluster/node/vmid # Disks mapping @@ -147,10 +157,10 @@ if __name__ == "__main__": ) """ Init Core thread """ - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": "Init Core thread"}) - core = Core(generalconf) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": "Init Core thread"}) + core = Core(generalconf, logger) """ Init API thread """ - logger.write({"result": "INFO", "type": "HYPERPROXMOX", "value": "Init API thread"}) - api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf) + logger.write({"thread":threading.get_ident(), "result": "INFO", "type": "HYPERPROXMOX", "value": "Init API thread"}) + api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf, logger) api_th.start() \ No newline at end of file