diff --git a/code/scripts/main/api/v1/api.py b/code/scripts/main/api/v1/api.py index 075915b..8bf8a0a 100644 --- a/code/scripts/main/api/v1/api.py +++ b/code/scripts/main/api/v1/api.py @@ -25,34 +25,65 @@ class Cluster: def GET(self, cluster=None): try: if cluster: - result = core.get_cluster(cluster) + result = { + "result": "OK", + "value": core.get_cluster(cluster) + } else: - result = core.get_cluster() + result = { + "result": "OK", + "value": core.get_cluster() + } except BaseException as e: - result = {"value": "{0} {1}".format("Invalid request", e)} + result = { + "result": "ERROR", + "type": "PYTHON - API", + "value": "{0} {1}".format("Invalid request:", e) + } return result def POST(self): try: data = json.loads(web.data().decode('utf-8')) - result = core.insert_cluster(data) + result = { + "result": "OK", + "value": core.insert_cluster(data) + } except BaseException as e: - result = {"value": "{0} {1}".format("Invalid request", e)} + result = { + "result": "ERROR", + "type": "PYTHON - API", + "value": "{0} {1}".format("Invalid request:", e) + } return result def PUT(self, cluster): try: data = json.loads(web.data().decode('utf-8')) - result = core.change_cluster(cluster, data) + result = { + "result": "OK", + "value": core.change_cluster(cluster, data) + } except BaseException as e: - result = {"value": "{0} {1}".format("Invalid request", e)} + result = { + "result": "ERROR", + "type": "PYTHON - API", + "value": "{0} {1}".format("Invalid request:", e) + } return result def DELETE(self, cluster): try: - result = core.delete_cluste(cluster) + result = { + "result": "OK", + "value": core.delete_cluste(cluster) + } except BaseException as e: - result = {"value": "{0} {1}".format("Invalid request", e)} + result = { + "result": "ERROR", + "type": "PYTHON - API", + "value": "{0} {1}".format("Invalid request:", e) + } return result diff --git a/code/scripts/main/core/core.py b/code/scripts/main/core/core.py index 7c7c674..eb58056 100644 --- a/code/scripts/main/core/core.py +++ b/code/scripts/main/core/core.py @@ -40,7 +40,6 @@ class Core: """ LOAD REDIS """ self.redis_msg = Lredis - #self.redis_msg.co = self.redis_msg.connect() if self.mongo.client and self.redis_msg.connect(): self.mongo.db = self.mongo.client.db @@ -52,7 +51,7 @@ class Core: self.delayrounddeploy = generalconf["deploy"]["delayrounddeploy"] """ RUN THE ANALYZER IN DEDICATED THEARD""" - self.clusters_conf = self.mongo.get_clusters_conf() + self.clusters_conf = self.mongo.get_clusters_conf()["value"] thc = threading.Thread(name="Update statistics", target=RunAnalyse, @@ -69,12 +68,15 @@ class Core: """ Find cluster informations from node """ lastkeyvalid = self.mongo.get_last_datekey() node_informations = self.mongo.get_nodes_informations((int(lastkeyvalid["value"])), target) - cluster_informations = self.mongo.get_clusters_conf(node_informations["cluster"]) + cluster_informations = self.mongo.get_clusters_conf(node_informations["cluster"])["value"] proxmox_cluster_url = cluster_informations["url"] proxmox_cluster_port = cluster_informations["port"] - proxmox_cluster_user = pdecrypt(cluster_informations["user"],self.generalconf["keys"]["key_pvt"]) - proxmox_cluster_pwd = pdecrypt(cluster_informations["password"], self.generalconf["keys"]["key_pvt"]) + proxmox_cluster_user = pdecrypt(base64.b64decode(cluster["user"]), + self.generalconf["keys"]["key_pvt"])["data"].decode('utf-8') + + proxmox_cluster_pwd = pdecrypt(base64.b64decode(cluster["password"]), + self.generalconf["keys"]["key_pvt"])["data"].decode('utf-8') proxmox_template = cluster_informations["template"] proxmox_storage_disk = cluster_informations["storage_disk"] @@ -169,7 +171,7 @@ class Core: instance_informations = self.mongo.get_instance(vmid) """ Find cluster informations from node """ - cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster']) + cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])["value"] proxmox_cluster_url = cluster_informations["url"] proxmox_cluster_port = cluster_informations["port"] @@ -190,8 +192,12 @@ class Core: self.mongo.delete_instance(vmid) self.mongo.update_system_free_ip(instance_informations['ip']) - except BaseException: - result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")} + except IndexError as ierror: + result = { + "result": "ERROR", + "type": "PROXMOX - VALUES", + "value": "{0} is not a valid VMID: {1}".format(vmid, ierror) + } return result @@ -201,7 +207,7 @@ class Core: instance_informations = self.mongo.get_instance(vmid) """ Find cluster informations from node """ - cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster']) + cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])["value"] proxmox_cluster_url = cluster_informations["url"] proxmox_cluster_port = cluster_informations["port"] @@ -221,8 +227,12 @@ class Core: "lxc", vmid, action) - except IndexError: - result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")} + except IndexError as ierror: + result = { + "result": "ERROR", + "type": "PROXMOX - VALUES", + "value": "{0} is not a valid VMID: {1}".format(vmid, ierror) + } return result @@ -232,7 +242,7 @@ class Core: instance_informations = self.mongo.get_instance(vmid) """ Find cluster informations from node """ - cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster']) + cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])["value"] proxmox_cluster_url = cluster_informations["url"] proxmox_cluster_port = cluster_informations["port"] @@ -252,8 +262,12 @@ class Core: "lxc", vmid) - except IndexError: - result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")} + except IndexError as ierror: + result = { + "result": "ERROR", + "type": "PROXMOX - VALUES", + "value": "{0} is not a valid VMID: {1}".format(vmid, ierror) + } return result @@ -263,7 +277,7 @@ class Core: instance_informations = self.mongo.get_instance(vmid) """ Find cluster informations from node """ - cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster']) + cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])["value"] proxmox_cluster_url = cluster_informations["url"] proxmox_cluster_port = cluster_informations["port"] @@ -287,8 +301,12 @@ class Core: if result['result'] == "OK": self.mongo.update_instance(vmid, data) - except IndexError: - result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")} + except IndexError as ierror: + result = { + "result": "ERROR", + "type": "PROXMOX - VALUES", + "value": "{0} is not a valid VMID: {1}".format(vmid, ierror) + } return result @@ -300,14 +318,14 @@ class Core: def get_cluster(self, cluster=None): """ Find cluster informations from node """ - cluster_informations = self.mongo.get_clusters_conf(cluster) + cluster_informations = self.mongo.get_clusters_conf(cluster)["value"] return cluster_informations def insert_cluster(self, data): testdata = valid_cluster_data(data) if not testdata: - if not self.mongo.get_clusters_conf(data["name"]): + if not self.mongo.get_clusters_conf(data["name"])["value"]: data["user"] = base64.b64encode(pcrypt(data["user"], self.generalconf["keys"]["key_pvt"])["data"]).decode('utf-8') data["password"] = base64.b64encode(pcrypt(data["password"], self.generalconf["keys"]["key_pvt"])["data"]).decode('utf-8') new_cluster = self.mongo.insert_new_cluster(data) diff --git a/code/scripts/main/core/modules/mod_analyst.py b/code/scripts/main/core/modules/mod_analyst.py index 2ae6fd3..509a767 100644 --- a/code/scripts/main/core/modules/mod_analyst.py +++ b/code/scripts/main/core/modules/mod_analyst.py @@ -57,56 +57,58 @@ class Analyse: for cluster in self.clusters_conf: """ Decode data """ - user = pdecrypt(base64.b64decode(cluster["user"]), + proxmox_cluster_user = pdecrypt(base64.b64decode(cluster["user"]), self.generalconf["keys"]["key_pvt"])["data"].decode('utf-8') - password = pdecrypt(base64.b64decode(cluster["password"]), + proxmox_cluster_pwd = pdecrypt(base64.b64decode(cluster["password"]), self.generalconf["keys"]["key_pvt"])["data"].decode('utf-8') """ AUTH """ proxmox = Proxmox("Analyse") - proxmox.get_ticket("{0}:{1}".format(cluster["url"], int(cluster["port"])), user, password) + proxmox.get_ticket("{0}:{1}".format(cluster["url"], int(cluster["port"])), proxmox_cluster_user, proxmox_cluster_pwd) """ Get excluded nodes """ exclude_nodes = cluster["exclude_nodes"] """ UPDATE NODES LIST """ - nodes_list = proxmox.get_nodes("{0}:{1}".format(cluster["url"], int(cluster["port"])))["value"] + nodes_list = proxmox.get_nodes("{0}:{1}".format(cluster["url"], int(cluster["port"]))) + if nodes_list["result"] == "OK": + for value_nodes_list in nodes_list["value"]["data"]: + if value_nodes_list["node"] not in exclude_nodes: + """ TOTAL COUNT CPU and RAM allocate""" + list_instances = proxmox.get_instance("{0}:{1}".format(cluster["url"], int(cluster["port"])), + value_nodes_list["node"], "lxc")["value"] - for value_nodes_list in nodes_list["data"]: - if value_nodes_list["node"] not in exclude_nodes: - """ TOTAL COUNT CPU and RAM allocate""" - list_instances = proxmox.get_instance("{0}:{1}".format(cluster["url"], int(cluster["port"])), - value_nodes_list["node"], "lxc")["value"] + totalcpu = 0 + totalram = 0 + for key_list_instances, value_list_instances in list_instances.items(): + for instances in value_list_instances: + totalcpu = totalcpu + instances["cpus"] + totalram = totalram + instances["maxmem"] - totalcpu = 0 - totalram = 0 - for key_list_instances, value_list_instances in list_instances.items(): - for instances in value_list_instances: - totalcpu = totalcpu + instances["cpus"] - totalram = totalram + instances["maxmem"] + value_nodes_list["totalalloccpu"] = totalcpu + value_nodes_list["totalallocram"] = totalram + value_nodes_list["vmcount"] = len(list_instances.items()) - value_nodes_list["totalalloccpu"] = totalcpu - value_nodes_list["totalallocram"] = totalram - value_nodes_list["vmcount"] = len(list_instances.items()) + percent_cpu_alloc = (totalcpu / value_nodes_list["maxcpu"]) * 100 + percent_ram_alloc = (totalram / value_nodes_list["mem"]) * 100 - percent_cpu_alloc = (totalcpu / value_nodes_list["maxcpu"]) * 100 - percent_ram_alloc = (totalram / value_nodes_list["mem"]) * 100 + """ + weight of node = + (((Percent Alloc CPU x coef) + ( Percent Alloc RAM x coef)) / Total coef ) * Cluster weight + """ + weight = (((percent_cpu_alloc * 2) + (percent_ram_alloc * 4)) / 6) * int(cluster["weight"]) - """ - weight of node = - (((Percent Alloc CPU x coef) + ( Percent Alloc RAM x coef)) / Total coef ) * Cluster weight - """ - weight = (((percent_cpu_alloc * 2) + (percent_ram_alloc * 4)) / 6) * int(cluster["weight"]) + value_nodes_list["weight"] = int(weight) + value_nodes_list["date"] = int(insert_time) + value_nodes_list["cluster"] = cluster["name"] - value_nodes_list["weight"] = int(weight) - value_nodes_list["date"] = int(insert_time) - value_nodes_list["cluster"] = cluster["name"] + self.mongo.insert_node(value_nodes_list) - self.mongo.insert_node(value_nodes_list) + else: + print(nodes_list) self.mongo.update_datekey(int(insert_time), "OK") - return def set_attribution(self, count): diff --git a/code/scripts/main/core/modules/mod_database.py b/code/scripts/main/core/modules/mod_database.py index 02cf812..76d9393 100644 --- a/code/scripts/main/core/modules/mod_database.py +++ b/code/scripts/main/core/modules/mod_database.py @@ -110,26 +110,65 @@ class MongoDB: """ CLUSTER """ def get_clusters_conf(self, cluster=None): - if cluster: - return json.loads(dumps(self.db[self.collection_clusters].find_one({"name": cluster}))) - else: - return json.loads(dumps(self.db[self.collection_clusters].find({}))) + try: + if cluster: + result = { + "result": "OK", + "value": json.loads(dumps(self.db[self.collection_clusters].find_one({"name": cluster}))) + } + else: + result = { + "result": "OK", + "value": json.loads(dumps(self.db[self.collection_clusters].find({}))) + } + except BaseException as e: + result = { + "result": "ERROR", + "value": "{0} {1}".format("Invalid request", e) + } + return result def insert_new_cluster(self, data): try: self.db[self.collection_clusters].insert(data) - result = {"value": "{0} {1}".format(data["name"], "is now available")} + result = { + "result": "OK", + "value": "{0} {1}".format(data["name"], "is now available") + } except BaseException as e: - result = {"value": "{0} {1}".format("Invalid request", e)} - + result = { + "result": "ERROR", + "value": "{0} {1}".format("Invalid request", e) + } return result def update_cluster(self, cluster, data): - return self.db[self.collection_clusters].update({"vmid": str(cluster)}, {'$set': data}, upsert=False) + try: + self.db[self.collection_clusters].update({"vmid": str(cluster)}, {'$set': data}, upsert=False) + result = { + "result": "OK", + "value": "{0} {1}".format(data["name"], "has been updated") + } + except BaseException as e: + result = { + "result": "ERROR", + "value": "{0} {1}".format("Invalid request", e) + } + return result def delete_cluster(self, cluster): - return self.db[self.collection_clusters].remove({"cluster": str(cluster)}) - + try: + self.db[self.collection_clusters].remove({"cluster": str(cluster)}) + result = { + "result": "OK", + "value": "{0} {1}".format(cluster, "has been deleted") + } + except BaseException as e: + result = { + "result": "ERROR", + "value": "{0} {1}".format("Invalid request", e) + } + return result """ SYSTEM """ def get_system_info(self): diff --git a/code/scripts/main/core/modules/mod_proxmox.py b/code/scripts/main/core/modules/mod_proxmox.py index 3788709..d8834d7 100644 --- a/code/scripts/main/core/modules/mod_proxmox.py +++ b/code/scripts/main/core/modules/mod_proxmox.py @@ -47,16 +47,16 @@ class Proxmox: self.csrf = {'CSRFPreventionToken': self.ticket.json()['data']['CSRFPreventionToken']} else: result = {"result": "ERROR", - "target": "{0}".format(url), + "target": "{0}".format(request), "type": "PROXMOX - STATUS CODE", - "customerror": "Error nodes informations. Bad HTTP Status code : " + "value": "Error nodes informations. Bad HTTP Status code : " "{0} -- {1}".format(self.ticket.status_code, self.ticket.text) } except (TypeError, ValueError, requests.exceptions.RequestException) as e: result = {"result": "ERROR", "target": "{0}".format(url), "type": "PYTHON", - "customerror": "Cannot get ticket session {0} ({1})".format(url, e)} + "value": "Cannot get ticket session {0} ({1})".format(url, e)} return result @@ -79,18 +79,18 @@ class Proxmox: else: result = { "result": "ERROR", - "target": "{0}".format(url), + "target": "{0}".format(request), "type": "PROXMOX - STATUS CODE", - "customerror": "Error nodes informations. Bad HTTP Status code : " + "value": "Error nodes informations. Bad HTTP Status code : " "{0} -- {1}".format(nodes.status_code, nodes.text) } except (TypeError, ValueError, requests.exceptions.RequestException) as e: result = { "result": "ERROR", - "target": "{0}".format(url), + "target": "{0}".format(request), "type": "PYTHON", - "customerror": "Cannot get node information for {0} ({1})".format(url, e) + "value": "Cannot get node information for {0} ({1})".format(url, e) } return result @@ -107,7 +107,7 @@ class Proxmox: result = {"result": "OK"} except (TypeError, ValueError, requests.exceptions.RequestException) as e: result = {"result": "ERROR", "target": "[{3}]", - "customerror": "Cannot get node information for {0} ({1})".format(url, e, nodename)} + "value": "Cannot get node information for {0} ({1})".format(url, e, nodename)} return result @@ -123,7 +123,7 @@ class Proxmox: result = {"result": "OK"} except (TypeError, ValueError, requests.exceptions.RequestException) as e: result = {"result": "ERROR", "target": "[{3}]", - "customerror": "Cannot get storage information for {0} ({1})".format(url, e, nodename)} + "value": "Cannot get storage information for {0} ({1})".format(url, e, nodename)} return result @@ -140,7 +140,7 @@ class Proxmox: result = {"result": "OK"} except (TypeError, ValueError, requests.exceptions.RequestException) as e: result = {"result": "ERROR", "target": "[{3}]", - "customerror": "Cannot get disks information for {0} ({1})".format(url, e, nodename)} + "value": "Cannot get disks information for {0} ({1})".format(url, e, nodename)} return result @@ -167,7 +167,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(url), "type": "PROXMOX - STATUS CODE", - "customerror": "Error nodes informations. Bad HTTP Status code : " + "value": "Error nodes informations. Bad HTTP Status code : " "{0} -- {1}".format(instances.status_code, instances.text) } except (TypeError, ValueError, requests.exceptions.RequestException) as e: @@ -175,7 +175,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(url), "type": "PYTHON", - "customerror": "Cannot get VM information for {0} {1} ({2})".format(url, nodename, e) + "value": "Cannot get VM information for {0} {1} ({2})".format(url, nodename, e) } return result @@ -201,7 +201,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(url), "type": "PROXMOX - STATUS CODE", - "customerror": "Error nodes informations. Bad HTTP Status code : " + "value": "Error nodes informations. Bad HTTP Status code : " "{0} -- {1}".format(config.status_code, config.text) } except (TypeError, ValueError, requests.exceptions.RequestException) as e: @@ -209,7 +209,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(url), "type": "PYTHON", - "customerror": "Cannot get VM information for {0} {1} ({2})".format(url, nodename, e) + "value": "Cannot get VM information for {0} {1} ({2})".format(url, nodename, e) } return result @@ -241,7 +241,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PROXMOX - STATUS CODE", - "customerror": "Error creating Container. Bad HTTP Status code : " + "value": "Error creating Container. Bad HTTP Status code : " "{0} -- {1}".format(createvm.status_code, createvm.text) } @@ -249,7 +249,7 @@ class Proxmox: result = {"result": "ERROR", "target": "{0}".format(nodename), "type": "PYTHON", - "customerror": "Cannot create this instance on {0} : ({1})".format(url, e)} + "value": "Cannot create this instance on {0} : ({1})".format(url, e)} return result @@ -281,7 +281,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PROXMOX - STATUS CODE", - "customerror": "Error delete Container. Bad HTTP Status code : " + "value": "Error delete Container. Bad HTTP Status code : " "{0} -- {1}".format(deletevm.status_code, deletevm.text) } @@ -289,7 +289,7 @@ class Proxmox: result = {"result": "ERROR", "target": "{0}".format(nodename), "type": "PYTHON", - "customerror": "Cannot delete Container ({2}) on {0} : ({1})".format(url, e, vmid)} + "value": "Cannot delete Container ({2}) on {0} : ({1})".format(url, e, vmid)} return result @@ -326,7 +326,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PROXMOX - STATUS CODE", - "customerror": "Error action Container. Bad HTTP Status code : " + "value": "Error action Container. Bad HTTP Status code : " "{0} -- {1}".format(statusm.status_code, statusm.text) } @@ -334,7 +334,7 @@ class Proxmox: result = {"result": "ERROR", "target": "{0}".format(nodename), "type": "PYTHON", - "customerror": "Cannot do this action this instance ({2}) on {0} : ({1})".format(url, e, vmid)} + "value": "Cannot do this action this instance ({2}) on {0} : ({1})".format(url, e, vmid)} return result @@ -366,7 +366,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PROXMOX - STATUS CODE", - "customerror": "Error resizing container. Bad HTTP Status code : " + "value": "Error resizing container. Bad HTTP Status code : " "{0} -- {1}".format(resizevm.status_code, resizevm.text) } except (TypeError, ValueError, requests.exceptions.RequestException) as e: @@ -374,7 +374,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PYTHON", - "customerror": "Cannot resize this instance {2} on {0} : ({1})".format(url, e, instanceid) + "value": "Cannot resize this instance {2} on {0} : ({1})".format(url, e, instanceid) } return result @@ -401,7 +401,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PROXMOX - STATUS CODE", - "customerror": "Error to find statistic for instance {2}. Bad HTTP Status code : " + "value": "Error to find statistic for instance {2}. Bad HTTP Status code : " "{0} -- {1}".format(statsvm.status_code, statsvm.text, instanceid) } except (TypeError, ValueError, requests.exceptions.RequestException) as e: @@ -409,7 +409,7 @@ class Proxmox: "result": "ERROR", "target": "{0}".format(nodename), "type": "PYTHON", - "customerror": "Cannot find statistic for this instance {2} on {0} : ({1})".format(url, e, instanceid) + "value": "Cannot find statistic for this instance {2} on {0} : ({1})".format(url, e, instanceid) } return result