From d8be9643bbe5cbeb299361c7975eb3673f520ef1 Mon Sep 17 00:00:00 2001 From: Tlams Date: Fri, 9 Feb 2018 14:28:11 +0000 Subject: [PATCH] Many syntaxfix and debug --- code/scripts/main/api/v1/api.py | 43 +++++++++---------- code/scripts/main/core/core.py | 19 ++++++-- code/scripts/main/core/modules/mod_analyst.py | 17 +++++--- .../scripts/main/core/modules/mod_database.py | 20 ++++++--- code/scripts/main/core/modules/mod_proxmox.py | 2 +- code/scripts/main/startup.py | 2 +- 6 files changed, 60 insertions(+), 43 deletions(-) diff --git a/code/scripts/main/api/v1/api.py b/code/scripts/main/api/v1/api.py index 98dbf13..1db9f5b 100644 --- a/code/scripts/main/api/v1/api.py +++ b/code/scripts/main/api/v1/api.py @@ -43,7 +43,7 @@ class General_Search: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -55,27 +55,24 @@ class QueryCache_Infra: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result +class QueryDates: + def GET(self, keytype): + try: + result = core.getkey(keytype) + except BaseException as e: + result = { + "result": "ERROR", + "type": "PYTHON - API", + "value": "Invalid request: {0}".format(e) + } + return result -class Static_Nodes: - def GET(self, date, cluster=None, node=None): - if node and cluster: - return core.generalsearch('{ "date": {0}, "cluster": {1}, "node": {2} }'.format(date, cluster, node)) - elif cluster: - return core.generalsearch('{ "date": {0}, "cluster": {1} }'.format(date, cluster)) - else: - return core.generalsearch('{ "date": {0}}'.format(date)) - - -class Dates: - def GET(self): - return core.generalsearch('{ "_id": {id} }'.format(id=nodeid)) - """ CLASS DIRECT """ class Cluster: @@ -89,7 +86,7 @@ class Cluster: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -101,7 +98,7 @@ class Cluster: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -113,7 +110,7 @@ class Cluster: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -124,7 +121,7 @@ class Cluster: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -191,7 +188,7 @@ class Instance: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -203,7 +200,7 @@ class Instance: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result @@ -214,7 +211,7 @@ class Instance: result = { "result": "ERROR", "type": "PYTHON - API", - "value": "{0} {1}".format("Invalid request:", e) + "value": "Invalid request: {0}".format(e) } return result diff --git a/code/scripts/main/core/core.py b/code/scripts/main/core/core.py index acd228a..b0cee50 100644 --- a/code/scripts/main/core/core.py +++ b/code/scripts/main/core/core.py @@ -41,10 +41,13 @@ class Core: """ LOAD REDIS """ self.redis_msg = Redis_wrapper(generalconf["redis"]["ip"], - generalconf["redis"]["port"], 0).connect() + generalconf["redis"]["port"], 0) self.redis_cache = Redis_wrapper(generalconf["redis"]["ip"], - generalconf["redis"]["port"], 3).connect() + generalconf["redis"]["port"], 3) + + self.redis_msg.connect() + self.redis_cache.connect() if self.mongo.client and self.redis_msg: self.mongo.db = self.mongo.client.db @@ -87,13 +90,21 @@ class Core: except: return json_decode({"value": "Bad request"}) + def getkey(self, keytype): + if keytype == "all": + return self.mongo.get_all_datekey() + elif keytype == "last": + return self.mongo.get_last_datekey() + + + def generalquerycacheinfra(self, dest, date, cluster=None, node=None, vmid=None): """ Test Redis Cache """ - hash_object = hashlib.md5(b"{0}-{1}-{2}-{3}-{4}".format(dest, date, cluster, node, vmid)) + hash_object = hashlib.md5("{0}-{1}-{2}-{3}-{4}".format(dest, date, cluster, node, vmid).encode('utf-8')) hash_hex = hash_object.hexdigest() - cache = self.redis_cache.get_message(hash_hex) + cache = None # self.redis_cache.get_message(hash_hex) if cache is None: if dest == "instances": diff --git a/code/scripts/main/core/modules/mod_analyst.py b/code/scripts/main/core/modules/mod_analyst.py index 80c3cf0..7183f4b 100644 --- a/code/scripts/main/core/modules/mod_analyst.py +++ b/code/scripts/main/core/modules/mod_analyst.py @@ -86,13 +86,14 @@ class Analyse: clusters_status = proxmox.get_clusters("{0}:{1}".format(cluster["url"], int(cluster["port"]))) clusters_status["date"] = int(insert_time) clusters_status["cluster"] = cluster["name"] - self.mongo.insert_clusters(instance) + self.mongo.insert_clusters(clusters_status) """ UPDATE NODES LIST """ nodes_list = proxmox.get_nodes("{0}:{1}".format(cluster["url"], int(cluster["port"]))) if nodes_list["result"] == "OK": for value_nodes_list in nodes_list["value"]["data"]: # if value_nodes_list["node"] not in exclude_nodes: + list_instances = {} """ TOTAL COUNT CPU and RAM allocate """ if (instancetype == "all"): types = ["lxc", "qemu"] # vz... @@ -114,15 +115,16 @@ class Analyse: ############# """ for key_list_instances, value_list_instances in list_instances.items(): - for instances in value_list_instances: + for instance in value_list_instances: """ Update cpu and ram for node """ - totalcpu = totalcpu + instances["cpus"] - totalram = totalram + instances["maxmem"] + totalcpu = totalcpu + instance["cpus"] + totalram = totalram + instance["maxmem"] """ Update instance list """ instance["cluster"] = cluster["name"] instance["node"] = value_nodes_list["node"] instance["date"] = int(insert_time) + instance["type"] = "" # a revoir self.mongo.insert_instances(instance) """ @@ -162,16 +164,17 @@ class Analyse: """ storages_list = proxmox.get_storages("{0}:{1}".format(cluster["url"], int(cluster["port"])), value_nodes_list["node"]) - for storage in storages_list: + + for storage in storages_list["value"]["data"]: storage["node"] = value_nodes_list["node"] storage["date"] = int(insert_time) storage["cluster"] = cluster["name"] self.mongo.insert_storages(storage) - disks_list = proxmox.get_storages("{0}:{1}".format(cluster["url"], int(cluster["port"])), + disks_list = proxmox.get_disks("{0}:{1}".format(cluster["url"], int(cluster["port"])), value_nodes_list["node"], storage["storage"]) - for disk in disks_list: + for disk in disks_list["value"]["data"]: disk["storage"] = storage["storage"] disk["node"] = value_nodes_list["node"] disk["date"] = int(insert_time) diff --git a/code/scripts/main/core/modules/mod_database.py b/code/scripts/main/core/modules/mod_database.py index a0787e8..d75fa4f 100644 --- a/code/scripts/main/core/modules/mod_database.py +++ b/code/scripts/main/core/modules/mod_database.py @@ -98,14 +98,14 @@ class MongoDB: result = { "result": "OK", "value": json.loads( - dumps(self.db[self.collection_clusters].find({'date': date, 'grata': str(grata)}))) + dumps(self.db[self.collection_clusters].find({'date': int(date)}))) } else: result = { "result": "OK", "value": json.loads( dumps(self.db[self.collection_clusters].find_one( - {'$and': [{'date': date, 'cluster': cluster}]}))) + {'$and': [{'date': int(date), 'cluster': cluster}]}))) } except BaseException as serr: @@ -205,12 +205,17 @@ class MongoDB: return self.db[self.collection_datekey].insert({'date': int(date), 'status': status}) def update_datekey(self, date, status): - self.db[self.collection_datekey].update({'date': date}, {'$set': {'status': status}}, upsert=False) + self.db[self.collection_datekey].update({'date': int(date)}, {'$set': {'status': status}}, upsert=False) def get_last_datekey(self): - last_id = self.db[self.collection_datekey].find({'status': 'OK'}).sort("date", -1).limit(1) + last_id = self.db[self.collection_datekey].find({'status': 'OK'}, {"date": 1, "_id": 0}).sort("date", -1) return {"value": int(json.loads(dumps(last_id))[0]['date'])} + def get_all_datekey(self): + keylist = self.db[self.collection_datekey].find({'status': 'OK'}, + {"date": 1, "_id": 0}).sort("date", -1) + return {"value": json.loads(dumps(keylist))} + """ NODES MANAGEMENT""" def insert_nodes(self, data): try: @@ -225,21 +230,22 @@ class MongoDB: "result": "OK", "value": json.loads( dumps(self.db[self.collection_nodes].find( - {'$and': [{'date': date, 'grata': str(grata)}]}))) + {'date': int(date)}))) } + elif not node: result = { "result": "OK", "value": json.loads( dumps(self.db[self.collection_nodes].find( - {'$and': [{'date': date, 'cluster': cluster, 'grata': str(grata)}]}))) + {'$and': [{'date': int(date), 'cluster': cluster}]}))) } else: result = { "result": "OK", "value": json.loads( dumps(self.db[self.collection_nodes].find_one( - {'$and': [{'date': date, 'cluster': cluster, 'node': node, 'grata': str(grata)}]}))) + {'$and': [{'date': int(date), 'cluster': cluster, 'node': node}]}))) } except BaseException as serr: diff --git a/code/scripts/main/core/modules/mod_proxmox.py b/code/scripts/main/core/modules/mod_proxmox.py index f6c261e..7b7d0e8 100644 --- a/code/scripts/main/core/modules/mod_proxmox.py +++ b/code/scripts/main/core/modules/mod_proxmox.py @@ -104,7 +104,7 @@ class Proxmox: Get Nodes from cluster :param url: Generic node url (node = physical hypervisor) """ - request = "https://{0}/api2/json/clusters/status".format(url) + request = "https://{0}/api2/json/cluster/status".format(url) try: nodes = self.nodes = self.socket.get(request, cookies=self.PVEAuthCookie, diff --git a/code/scripts/main/startup.py b/code/scripts/main/startup.py index c0c7dae..a9a0e2a 100644 --- a/code/scripts/main/startup.py +++ b/code/scripts/main/startup.py @@ -122,7 +122,7 @@ if __name__ == "__main__": '/api/v1/static/(clusters)/([0-9]+)/', 'QueryCache_Infra', # date - '/api/v1/static/dates/', 'QueryCache_Dates', + '/api/v1/static/dates/(all|last)', 'QueryDates', # mongoid '/api/v1/static/(instances|nodes|clusters|storages|disks)/id/([a-z0-9]+)', 'General_Search',