external/HyperProxmox
1
0
Fork 0
mirror of https://github.com/ThomasGsp/HyperProxmox.git synced 2025-03-09 15:40:18 +00:00
Code Issues Releases Wiki Activity

Changes for redis

Browse source 
- Cache system
- Internal working
- Code clean
This commit is contained in:
Tlams 2018-02-08 14:30:00 +00:00
parent 78fca2bef4
commit a7d66036c0
5 changed files with 52 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

11
code/scripts/main/api/v1/api.py
View file

@ -180,13 +180,13 @@ class Instance:
for t in range(timeout):
time.sleep(1)
try:
if len(ast.literal_eval(Lredis.get_message(command_id))) == int(count):
if len(ast.literal_eval(redis_msg.get_message(command_id))) == int(count):
break
except BaseException as err:
print("Value not found", err)
""" Return messages """
return ast.literal_eval(Lredis.get_message(command_id))
return ast.literal_eval(redis_msg.get_message(command_id))
except BaseException as e:
result = {
"result": "ERROR",
@ -220,12 +220,13 @@ class Instance:
class ThreadAPI(threading.Thread):
def __init__(self, threadid, name, urls, c, g, r):
#def __init__(self, threadid, name, urls, c, g, r):
def __init__(self, threadid, name, urls, c, g):
""" Pass Global var in this theard."""
global core, generalconf, Lredis
global core, generalconf, redis_msg
core = c
generalconf = g
Lredis = r
redis_msg = core.redis_msg
""" RUN API """
threading.Thread.__init__(self)

42
code/scripts/main/core/core.py
View file

@ -16,6 +16,7 @@ from netaddr import iter_iprange
import threading
import time
import base64
import hashlib
def RunAnalyse(clusters_conf, generalconf, delay=300):
@ -29,19 +30,23 @@ def RunAnalyse(clusters_conf, generalconf, delay=300):
class Core:
def __init__(self, generalconf, Lredis):
# def __init__(self, generalconf, Lredis):
def __init__(self, generalconf):
self.generalconf = generalconf
self.Lredis = Lredis
""" LOAD MONGODB """
self.mongo = MongoDB(generalconf["mongodb"]["ip"])
self.mongo.client = self.mongo.connect()
""" LOAD REDIS """
self.redis_msg = Lredis
self.redis_msg = Redis_wrapper(generalconf["redis"]["ip"],
generalconf["redis"]["port"], 0).connect()
if self.mongo.client and self.redis_msg.connect():
self.redis_cache = Redis_wrapper(generalconf["redis"]["ip"],
generalconf["redis"]["port"], 3).connect()
if self.mongo.client and self.redis_msg:
self.mongo.db = self.mongo.client.db
""" Others """
@ -78,15 +83,27 @@ class Core:
return json_decode({"value": "Bad request"})
def generalquerycacheinfra(self, dest, date, cluster=None, node=None, vmid=None):
if dest == "instances":
return self.mongo.get_instance(date, cluster, node, vmid)
elif dest == "nodes":
return self.mongo.get_nodes_informations(date, cluster, node)
elif dest == "clusters":
return self.mongo.get_clusters_conf(date, cluster)
else:
json_decode({"value": "Bad request"})
""" Test Redis Cache """
hash_object = hashlib.md5(b"{0}-{1}-{2}-{3}-{4}".format(dest, date, cluster, node, vmid))
hash_hex = hash_object.hexdigest()
cache = self.redis_cache.get_message(hash_hex)
if cache is None:
if dest == "instances":
resultmbrequest = self.mongo.get_instance(date, cluster, node, vmid)
elif dest == "nodes":
resultmbrequest = self.mongo.get_nodes_informations(date, cluster, node)
elif dest == "clusters":
resultmbrequest = self.mongo.get_clusters_conf(date, cluster)
else:
resultmbrequest = json_decode({"value": "Bad request"})
self.redis_cache.insert_message(hash_hex, resultmbrequest)
return resultmbrequest
else:
return cache
"""
@ -189,6 +206,7 @@ class Core:
data["cluster"] = node_informations["cluster"]
data["node"] = target
data["ip"] = ip
data["date"] = lastkeyvalid["value"]
self.mongo.insert_instance(data)
""" Limit creation DDOS based on digest """

50
code/scripts/main/core/modules/mod_database.py
View file

@ -4,8 +4,14 @@ import json
import redis
import time
class Redis_instance_queue:
def __init__(self, server="127.0.0.1", port=6379, db=3, password=None):
class Redis_wrapper:
def __init__(self, server="127.0.0.1", port=6379, db=0, password=None):
# DB =
# messages: 0
# logs: 1
# queue : 2
# cache : 3
self.server = server
self.port = port
self.r = None
@ -26,49 +32,9 @@ class Redis_instance_queue:
def insert_instance_queue(self, logtext, expir=3000):
self.r.set(time.time(), logtext, expir)
class Redis_logger:
def __init__(self, server="127.0.0.1", port=6379, db=2, password=None):
self.server = server
self.port = port
self.r = None
self.db = db
self.password = password
def connect(self):
try:
conn = self.r = redis.Redis(
host=self.server, port=self.port, db=self.db, password=self.password,
charset="utf-8", decode_responses=True)
self.r.client_list()
except BaseException as err:
print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err))
conn = False
return conn
def insert_logs(self, logtext, expir=86400*4):
self.r.set(time.time(), logtext, expir)
class Redis_messages:
def __init__(self, server="127.0.0.1", port=6379, db=1, password=None):
self.server = server
self.port = port
self.r = None
self.db = db
self.password = password
def connect(self):
try:
conn = self.r = redis.Redis(
host=self.server, port=self.port, db=self.db, password=self.password,
charset="utf-8", decode_responses=True)
self.r.client_list()
except BaseException as err:
print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err))
conn = False
return conn
def insert_message(self, key, value, expir=86400):
self.r.set(key, value, expir)

4
code/scripts/main/private/conf/config
View file

@ -17,12 +17,12 @@ user: hosting_api
; NOSQL databases, should use a password
mongodb_user:
mongodb_password:
mongodb_ip: 192.168.66.1
mongodb_ip: 192.168.0.138
mongodb_port: 27017
redis_user:
redis_password:
redis_ip: 192.168.66.1
redis_ip: 192.168.0.138
redis_port: 6379
[options]

10
code/scripts/main/startup.py
View file

@ -121,13 +121,15 @@ if __name__ == "__main__":
}
""" First redis connection """
Lredis = Redis_messages(generalconf["redis"]["ip"])
Lredis.connect()
# Lredis = Redis_wrapper(generalconf["redis"]["ip"], generalconf["redis"]["port"])
# Lredis.connect()
""" Init Core thread """
core = Core(generalconf, Lredis)
# core = Core(generalconf, Lredis)
core = Core(generalconf)
""" Init API thread """
api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf, Lredis)
# api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf, Lredis)
api_th = ThreadAPI(1, "ThreadAPI", urls, core, generalconf)
api_th.start()