diff --git a/code/scripts/main/api/v1/api.py b/code/scripts/main/api/v1/api.py index f0971e3..2e245ec 100644 --- a/code/scripts/main/api/v1/api.py +++ b/code/scripts/main/api/v1/api.py @@ -180,13 +180,13 @@ class Instance: for t in range(timeout): time.sleep(1) try: - if len(ast.literal_eval(Lredis.get_message(command_id))) == int(count): + if len(ast.literal_eval(redis_msg.get_message(command_id))) == int(count): break except BaseException as err: print("Value not found", err) """ Return messages """ - return ast.literal_eval(Lredis.get_message(command_id)) + return ast.literal_eval(redis_msg.get_message(command_id)) except BaseException as e: result = { "result": "ERROR", @@ -220,12 +220,13 @@ class Instance: class ThreadAPI(threading.Thread): - def __init__(self, threadid, name, urls, c, g, r): + #def __init__(self, threadid, name, urls, c, g, r): + def __init__(self, threadid, name, urls, c, g): """ Pass Global var in this theard.""" - global core, generalconf, Lredis + global core, generalconf, redis_msg core = c generalconf = g - Lredis = r + redis_msg = core.redis_msg """ RUN API """ threading.Thread.__init__(self) diff --git a/code/scripts/main/core/core.py b/code/scripts/main/core/core.py index 5304438..3626343 100644 --- a/code/scripts/main/core/core.py +++ b/code/scripts/main/core/core.py @@ -16,6 +16,7 @@ from netaddr import iter_iprange import threading import time import base64 +import hashlib def RunAnalyse(clusters_conf, generalconf, delay=300): @@ -29,19 +30,23 @@ def RunAnalyse(clusters_conf, generalconf, delay=300): class Core: - def __init__(self, generalconf, Lredis): + # def __init__(self, generalconf, Lredis): + def __init__(self, generalconf): self.generalconf = generalconf - self.Lredis = Lredis """ LOAD MONGODB """ self.mongo = MongoDB(generalconf["mongodb"]["ip"]) self.mongo.client = self.mongo.connect() """ LOAD REDIS """ - self.redis_msg = Lredis + self.redis_msg = Redis_wrapper(generalconf["redis"]["ip"], + generalconf["redis"]["port"], 0).connect() - if self.mongo.client and self.redis_msg.connect(): + self.redis_cache = Redis_wrapper(generalconf["redis"]["ip"], + generalconf["redis"]["port"], 3).connect() + + if self.mongo.client and self.redis_msg: self.mongo.db = self.mongo.client.db """ Others """ @@ -78,15 +83,27 @@ class Core: return json_decode({"value": "Bad request"}) def generalquerycacheinfra(self, dest, date, cluster=None, node=None, vmid=None): - if dest == "instances": - return self.mongo.get_instance(date, cluster, node, vmid) - elif dest == "nodes": - return self.mongo.get_nodes_informations(date, cluster, node) - elif dest == "clusters": - return self.mongo.get_clusters_conf(date, cluster) - else: - json_decode({"value": "Bad request"}) + """ Test Redis Cache """ + hash_object = hashlib.md5(b"{0}-{1}-{2}-{3}-{4}".format(dest, date, cluster, node, vmid)) + hash_hex = hash_object.hexdigest() + + cache = self.redis_cache.get_message(hash_hex) + + if cache is None: + if dest == "instances": + resultmbrequest = self.mongo.get_instance(date, cluster, node, vmid) + elif dest == "nodes": + resultmbrequest = self.mongo.get_nodes_informations(date, cluster, node) + elif dest == "clusters": + resultmbrequest = self.mongo.get_clusters_conf(date, cluster) + else: + resultmbrequest = json_decode({"value": "Bad request"}) + + self.redis_cache.insert_message(hash_hex, resultmbrequest) + return resultmbrequest + else: + return cache """ @@ -189,6 +206,7 @@ class Core: data["cluster"] = node_informations["cluster"] data["node"] = target data["ip"] = ip + data["date"] = lastkeyvalid["value"] self.mongo.insert_instance(data) """ Limit creation DDOS based on digest """ diff --git a/code/scripts/main/core/modules/mod_database.py b/code/scripts/main/core/modules/mod_database.py index 1837385..e33e29c 100644 --- a/code/scripts/main/core/modules/mod_database.py +++ b/code/scripts/main/core/modules/mod_database.py @@ -4,8 +4,14 @@ import json import redis import time -class Redis_instance_queue: - def __init__(self, server="127.0.0.1", port=6379, db=3, password=None): +class Redis_wrapper: + def __init__(self, server="127.0.0.1", port=6379, db=0, password=None): + # DB = + # messages: 0 + # logs: 1 + # queue : 2 + # cache : 3 + self.server = server self.port = port self.r = None @@ -26,49 +32,9 @@ class Redis_instance_queue: def insert_instance_queue(self, logtext, expir=3000): self.r.set(time.time(), logtext, expir) - -class Redis_logger: - def __init__(self, server="127.0.0.1", port=6379, db=2, password=None): - self.server = server - self.port = port - self.r = None - self.db = db - self.password = password - - def connect(self): - try: - conn = self.r = redis.Redis( - host=self.server, port=self.port, db=self.db, password=self.password, - charset="utf-8", decode_responses=True) - self.r.client_list() - except BaseException as err: - print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err)) - conn = False - return conn - def insert_logs(self, logtext, expir=86400*4): self.r.set(time.time(), logtext, expir) - -class Redis_messages: - def __init__(self, server="127.0.0.1", port=6379, db=1, password=None): - self.server = server - self.port = port - self.r = None - self.db = db - self.password = password - - def connect(self): - try: - conn = self.r = redis.Redis( - host=self.server, port=self.port, db=self.db, password=self.password, - charset="utf-8", decode_responses=True) - self.r.client_list() - except BaseException as err: - print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err)) - conn = False - return conn - def insert_message(self, key, value, expir=86400): self.r.set(key, value, expir) diff --git a/code/scripts/main/private/conf/config b/code/scripts/main/private/conf/config index 3a3e527..b515f52 100644 --- a/code/scripts/main/private/conf/config +++ b/code/scripts/main/private/conf/config @@ -17,12 +17,12 @@ user: hosting_api ; NOSQL databases, should use a password mongodb_user: mongodb_password: -mongodb_ip: 192.168.66.1 +mongodb_ip: 192.168.0.138 mongodb_port: 27017 redis_user: redis_password: -redis_ip: 192.168.66.1 +redis_ip: 192.168.0.138 redis_port: 6379 [options] diff --git a/code/scripts/main/startup.py b/code/scripts/main/startup.py index 1b9400e..d8b0f66 100644 --- a/code/scripts/main/startup.py +++ b/code/scripts/main/startup.py @@ -121,13 +121,15 @@ if __name__ == "__main__": } """ First redis connection """ - Lredis = Redis_messages(generalconf["redis"]["ip"]) - Lredis.connect() + # Lredis = Redis_wrapper(generalconf["redis"]["ip"], generalconf["redis"]["port"]) + # Lredis.connect() """ Init Core thread """ - core = Core(generalconf, Lredis) + # core = Core(generalconf, Lredis) + core = Core(generalconf) """ Init API thread """ - api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf, Lredis) + # api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf, Lredis) + api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf) api_th.start()