first commit

code/scripts/main/core/README.md

@@ -0,0 +1,3 @@
+##### Core funtions
+
+The core is divised manage load differents modules with the goal to manage then and orchestrate all function.

code/scripts/main/core/core.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+ Author: Tlams
+ Langage: Python
+ Minimum version require: 3.4
+"""
+
+from core.modules.mod_proxmox import *
+from core.modules.mod_database import *
+from core.modules.mod_analyst import *
+from core.modules.mod_access import *
+from core.libs.hcrypt import *
+from netaddr import iter_iprange
+import threading
+import time
+
+
+def RunAnalyse(clusters_conf, generalconf, delay=300):
+    play = Analyse(clusters_conf, generalconf)
+
+    while True:
+        play.run()
+        time.sleep(delay)
+
+
+class Core:
+
+    def __init__(self, generalconf, Lredis):
+
+        self.generalconf = generalconf
+        self.Lredis = Lredis
+
+        """ LOAD MONGODB """
+        self.mongo = MongoDB(generalconf["mongodb"]["ip"])
+        self.mongo.client = self.mongo.connect()
+
+        """ LOAD REDIS """
+        self.redis_msg = Lredis
+        #self.redis_msg.co = self.redis_msg.connect()
+
+        if self.mongo.client and self.redis_msg.connect():
+            self.mongo.db = self.mongo.client.db
+
+            """ Others """
+            # value
+            self.concurrencydeploy = generalconf["deploy"]["concurrencydeploy"]
+            # in seconds
+            self.delayrounddeploy = generalconf["deploy"]["delayrounddeploy"]
+
+            """ RUN THE ANALYZER IN DEDICATED THEARD"""
+            self.clusters_conf = self.mongo.get_clusters_conf()
+
+            thc = threading.Thread(name="Update statistics",
+                                   target=RunAnalyse,
+                                   args=(self.clusters_conf, self.generalconf))
+            thc.start()
+
+    """ 
+    #######################
+    # INSTANCE MANAGEMENT #
+    #######################
+    """
+    def insert_instance(self, target, count=1, command_id=000000):
+
+        """ Find cluster informations from node """
+        lastkeyvalid = self.mongo.get_last_datekey()
+        node_informations = self.mongo.get_nodes_informations((int(lastkeyvalid["value"])), target)
+        cluster_informations = self.mongo.get_clusters_conf(node_informations["cluster"])
+
+        proxmox_cluster_url = cluster_informations["url"]
+        proxmox_cluster_port = cluster_informations["port"]
+        proxmox_cluster_user = pdecrypt(cluster_informations["user"],self.generalconf["keys"]["key_pvt"])
+        proxmox_cluster_pwd = pdecrypt(cluster_informations["password"], self.generalconf["keys"]["key_pvt"])
+
+        proxmox_template = cluster_informations["template"]
+        proxmox_storage_disk = cluster_informations["storage_disk"]
+
+        """ LOAD PROXMOX """
+        proxmox = Proxmox(target)
+
+        proxmox.get_ticket("{0}:{1}".format(proxmox_cluster_url,
+                                                 int(proxmox_cluster_port)),
+                           proxmox_cluster_user,
+                           proxmox_cluster_pwd)
+
+        returnlistresult = []
+        currentcount = 0
+        for c in range(0, int(count)):
+
+            if currentcount == self.concurrencydeploy:
+                time.sleep(self.delayrounddeploy)
+                currentcount = 0
+
+            currentcount = currentcount + 1
+
+            get_info_system = self.mongo.get_system_info()
+            """ FIND NEXT INSTANCE ID AVAILABLE  AND INCREMENT IT"""
+            next_instance_id = int(get_info_system["instances_number"]+1)
+
+            """ FIND LAST LAST IP  USE  AND INCREMENT IT"""
+            if not get_info_system["IP_free"]:
+                get_instance_ip = get_info_system["IP_current"]
+                next_ip = iter_iprange(get_instance_ip, '172.16.255.250', step=1)
+                # Revoir pour un truc plus clean ....
+                next(next_ip)
+                ip = str(next(next_ip))
+            else:
+                ip = str(get_info_system["IP_free"][0])
+                self.mongo.update_system_delete_ip(ip)
+
+
+            """ INSTANCE DEFINITION """
+            data = {
+                'ostemplate': proxmox_template,
+                'vmid': next_instance_id,
+                'storage': proxmox_storage_disk,
+                'cores': 1,
+                'cpulimit': 1,
+                'cpuunits': 512,
+                'arch': "amd64",
+                'memory': 256,
+                'description': command_id,
+                'onboot': 0,
+                'swap': 256,
+                'ostype': 'debian',
+                'net0': 'name=eth0,bridge=vmbr1,ip={0}/16,gw=172.16.1.254'.format(ip),
+                'ssh-public-keys': get_info_system["sshpublickey"]
+            }
+
+            """ INSTANCE INSERTION """
+            result_new = {}
+            #while not proxmox.retry_on_errorcode(result_new['result']):
+            result_new = proxmox.create_instance("{0}:{1}".format(proxmox_cluster_url,
+                                                     int(proxmox_cluster_port)), target, "lxc",
+                                                    data)
+
+            """ VERIFY THE RESULT BY PROXMOX STATUS REQUEST CODE """
+            if result_new['result'] == "OK":
+                """ INCREMENT INSTANCE ID IN DATABASE """
+                self.mongo.update_system_instance_id(next_instance_id)
+                """ INCREMENT INSTANCE IP IN DATABASE """
+                self.mongo.update_system_instance_ip(ip)
+
+                """ INSERT THIS NEW SERVER IN DATABASE """
+                data["commandid"] = command_id
+                data["cluster"] = node_informations["cluster"]
+                data["node"] = target
+                data["ip"] = ip
+
+                self.mongo.insert_instance(data)
+                """ BREAK the loop due to valid creation """
+
+
+            returnlistresult.append(result_new)
+
+        """ SEND MESSAGE IN REDIS """
+        self.redis_msg.insert_message(command_id, returnlistresult)
+
+        return
+
+    def delete_instance(self, vmid):
+
+        try:
+            """ Find node/cluster informations from vmid """
+            instance_informations = self.mongo.get_instance(vmid)
+
+            """ Find cluster informations from node """
+            cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])
+
+            proxmox_cluster_url = cluster_informations["url"]
+            proxmox_cluster_port = cluster_informations["port"]
+            proxmox_cluster_user = pdecrypt(cluster_informations["user"],self.generalconf["keys"]["key_pvt"])
+            proxmox_cluster_pwd = pdecrypt(cluster_informations["password"], self.generalconf["keys"]["key_pvt"])
+
+            """ LOAD PROXMOX """
+            proxmox = Proxmox(instance_informations['node'])
+            proxmox.get_ticket("{0}:{1}".format(proxmox_cluster_url,
+                                                int(proxmox_cluster_port)),
+                               proxmox_cluster_user,
+                               proxmox_cluster_pwd)
+
+            result = proxmox.delete_instance("{0}:{1}".format(proxmox_cluster_url,
+                                                              int(proxmox_cluster_port)), instance_informations['node'], "lxc", vmid)
+
+            if result['result'] == "OK":
+                self.mongo.delete_instance(vmid)
+                self.mongo.update_system_free_ip(instance_informations['ip'])
+
+        except BaseException:
+            result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")}
+
+        return result
+
+    def status_instance(self, vmid, action):
+        """ Find node/cluster informations from vmid """
+        try:
+            instance_informations = self.mongo.get_instance(vmid)
+
+            """ Find cluster informations from node """
+            cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])
+
+            proxmox_cluster_url = cluster_informations["url"]
+            proxmox_cluster_port = cluster_informations["port"]
+            proxmox_cluster_user = pdecrypt(cluster_informations["user"],self.generalconf["keys"]["key_pvt"])
+            proxmox_cluster_pwd = pdecrypt(cluster_informations["password"], self.generalconf["keys"]["key_pvt"])
+
+            """ LOAD PROXMOX """
+            proxmox = Proxmox(instance_informations['node'])
+            proxmox.get_ticket("{0}:{1}".format(proxmox_cluster_url,
+                                                     int(proxmox_cluster_port)),
+                               proxmox_cluster_user,
+                               proxmox_cluster_pwd)
+
+            result = proxmox.status_instance("{0}:{1}".format(proxmox_cluster_url,
+                                                              int(proxmox_cluster_port)),
+                                             instance_informations['node'],
+                                             "lxc",
+                                             vmid, action)
+
+        except IndexError:
+            result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")}
+
+        return result
+
+    def info_instance(self, vmid):
+        """ Find node/cluster informations from vmid """
+        try:
+            instance_informations = self.mongo.get_instance(vmid)
+
+            """ Find cluster informations from node """
+            cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])
+
+            proxmox_cluster_url = cluster_informations["url"]
+            proxmox_cluster_port = cluster_informations["port"]
+            proxmox_cluster_user = pdecrypt(cluster_informations["user"],self.generalconf["keys"]["key_pvt"])
+            proxmox_cluster_pwd = pdecrypt(cluster_informations["password"], self.generalconf["keys"]["key_pvt"])
+
+            """ LOAD PROXMOX """
+            proxmox = Proxmox(instance_informations['node'])
+            proxmox.get_ticket("{0}:{1}".format(proxmox_cluster_url,
+                                                     int(proxmox_cluster_port)),
+                               proxmox_cluster_user,
+                               proxmox_cluster_pwd)
+
+            result = proxmox.get_config("{0}:{1}".format(proxmox_cluster_url,
+                                                              int(proxmox_cluster_port)),
+                                             instance_informations['node'],
+                                             "lxc",
+                                             vmid)
+
+        except IndexError:
+            result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")}
+
+        return result
+
+    def change_instance(self, vmid, data):
+        """ Find node/cluster informations from vmid """
+        try:
+            instance_informations = self.mongo.get_instance(vmid)
+
+            """ Find cluster informations from node """
+            cluster_informations = self.mongo.get_clusters_conf(instance_informations['cluster'])
+
+            proxmox_cluster_url = cluster_informations["url"]
+            proxmox_cluster_port = cluster_informations["port"]
+            proxmox_cluster_user = pdecrypt(cluster_informations["user"],self.generalconf["keys"]["key_pvt"])
+            proxmox_cluster_pwd = pdecrypt(cluster_informations["password"], self.generalconf["keys"]["key_pvt"])
+
+
+            """ LOAD PROXMOX """
+            proxmox = Proxmox(instance_informations['node'])
+            proxmox.get_ticket("{0}:{1}".format(proxmox_cluster_url,
+                                                int(proxmox_cluster_port)),
+                               proxmox_cluster_user,
+                               proxmox_cluster_pwd)
+
+            result = proxmox.resize_instance("{0}:{1}".format(proxmox_cluster_url,
+                                                              int(proxmox_cluster_port)),
+                                             instance_informations['node'],
+                                             "lxc",
+                                             vmid, data)
+
+            if result['result'] == "OK":
+                self.mongo.update_instance(vmid, data)
+
+        except IndexError:
+            result = {"value": "{0} {1}".format(vmid, "is not a valid VMID")}
+
+        return result
+
+    """ 
+    #######################
+    # CLUSTERS MANAGEMENT #
+    #######################
+    """
+
+    def get_cluster(self, cluster=None):
+        """ Find cluster informations from node """
+        cluster_informations = self.mongo.get_clusters_conf(cluster)
+        return cluster_informations
+
+    def insert_cluster(self, data):
+        testdata = valid_cluster_data(data)
+        if not testdata:
+            data["user"] = pcrypt(data["user"], self.generalconf["keys"]["key_pvt"])["data"]
+            data["password"] = pcrypt(data["password"], self.generalconf["keys"]["key_pvt"])["data"]
+            new_cluster = self.mongo.insert_new_cluster(data)
+        else:
+            new_cluster = {"error": "{1} {0}".format(testdata, "Invalid or miss paramettrer")}
+        return new_cluster
+
+    def change_cluster(self, cluster, data):
+        return
+
+    def delete_cluster(self, cluster):
+        return
+
+
+def valid_cluster_data(data):
+    key_required = ["name", "url", "port", "user", "password", "template", "storage_disk", "weight", "exclude_nodes"]
+    result = []
+    for key in key_required:
+        if key not in data:
+            result.append(key)
+    return result

code/scripts/main/core/libs/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-

code/scripts/main/core/libs/hcrypt.py

@@ -0,0 +1,11 @@
+from core.modules.mod_access import *
+
+def pcrypt(data, key):
+    CritConf = CryticalData()
+    data = CritConf.data_encryption(data, key)
+    return data
+
+def pdecrypt(data, key):
+    CritConf = CryticalData()
+    data = CritConf.data_decryption(data, key)
+    return data

code/scripts/main/core/modules/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-

code/scripts/main/core/modules/mod_access.py

@@ -0,0 +1,133 @@
+"""
+Author: Tlams
+Langage: Python
+Minimum version require: 3.4
+
+"""
+
+import os
+from Crypto.PublicKey import RSA
+import hashlib
+import codecs
+
+def encodepassphrase(passphrase):
+    return hashlib.sha512(passphrase.encode("UTF-8")).hexdigest()
+
+
+class CryticalData:
+    def __init__(self):
+        self.public_key = None
+        self.private_key = None
+
+    def generate_key(self, key_pvt, key_pub, passphrase, lgt=4096):
+        try:
+            private_key = RSA.generate(lgt)
+            file = open(key_pvt, "wb")
+            file.write(private_key.exportKey('PEM', passphrase, pkcs=1))
+            file.close()
+
+            public_key = private_key.publickey()
+            file = open(key_pub, "wb")
+            file.write(public_key.exportKey())
+            file.close()
+
+            os.chmod(key_pvt, 0o600)
+            os.chmod(key_pub, 0o600)
+            key_generation = {"result": "OK"}
+
+        except BaseException as e:
+            try:
+                print("Clean...")
+                os.remove(key_pvt)
+                os.remove(key_pub)
+            except OSError:
+                pass
+
+            key_generation = {
+                "result": "ERROR",
+                "type": "PYTHON",
+                "Error": "Key generation fail: {0}".format(e)
+            }
+
+        return key_generation
+
+    def read_public_key(self, key_pub):
+        try:
+            file_key_pub = open(key_pub, "rb")
+            self.public_key = RSA.importKey(file_key_pub.read())
+            file_key_pub.close()
+            result_public_key = {
+                "result": "OK",
+                "data": self.public_key
+            }
+        except BaseException as e:
+            result_public_key = {
+                "result": "ERROR",
+                "type": "PYTHON",
+                "error": "Your public key seem to invalid: {0}".format(e)
+            }
+        return result_public_key
+
+    def read_private_key(self, key_pvt, passphrase):
+        try:
+            file_key_pvt = open(key_pvt, "rb")
+            self.private_key = RSA.importKey(file_key_pvt.read(), passphrase)
+            file_key_pvt.close()
+            result_private_key = {
+                "result": "OK",
+                "data": self.private_key
+
+            }
+        except BaseException as e:
+            result_private_key = {
+                "result": "ERROR",
+                "type": "PYTHON",
+                "error": "Your private key seem to invalid: {0}".format(e)
+            }
+        return result_private_key
+
+    def data_encryption(self, data, key=None):
+
+        encfrypt = key.encrypt(data.encode("utf-8"), 32)
+        convert_str = str(encfrypt[0])[2:-1]
+
+
+        try:
+            if key:
+                result_encrypt = {
+                    "result": "OK",
+                    "data":  convert_str
+                }
+            else:
+                result_encrypt = {
+                    "result": "OK",
+                    "data":  codecs.encode(self.public_key.encrypt(mutable_bytes, 32)[0], 'base64')
+                }
+        except BaseException as e:
+            result_encrypt = {
+                "result": "ERROR",
+                "type": "PYTHON",
+                "error": "Data encryption failed: {0}".format(e)
+            }
+        return result_encrypt
+
+    def data_decryption(self, data, key=None):
+        try:
+            if key:
+                result_decryption = {
+                    "result": "OK",
+                    "data": key.decrypt(data)
+                }
+            else:
+                result_decryption = {
+                    "result": "OK",
+                    "data": self.private_key.decrypt(data)
+                }
+        except BaseException as e:
+            result_decryption = {
+                "result": "ERROR",
+                "type": "PYTHON",
+                "error": "Data decryption failed: {0}".format(e)
+            }
+        return result_decryption
+

code/scripts/main/core/modules/mod_analyst.py

@@ -0,0 +1,157 @@
+"""
+Author: Tlams
+Langage: Python
+Minimum version require: 3.4
+
+Module function:
+The goal of this module is to analyse the differents clusters and node
+to allocate news instances.
+"""
+
+from core.modules.mod_proxmox import *
+from core.modules.mod_database import *
+from core.libs.hcrypt import *
+import time
+import operator
+import random
+import codecs
+
+def add_token(tokens_in_slots, slot_distributions):
+    num_tokens = sum(tokens_in_slots)
+    if not num_tokens:
+        #first token can go anywhere
+        tokens_in_slots[random.randint(0, 2)] += 1
+        return
+    expected_tokens = [num_tokens*distr for distr in slot_distributions]
+    errors = [expected - actual
+              for expected, actual in zip(expected_tokens, tokens_in_slots)]
+    most_error = max(enumerate(errors), key=lambda i_e: i_e[1])
+    tokens_in_slots[most_error[0]] += 1
+
+
+def distribution(n, tokens_in_slots, slot_distributions):
+    for i in range(n):
+        add_token(tokens_in_slots, slot_distributions)
+    return tokens_in_slots
+
+
+class Analyse:
+    def __init__(self, clusters_conf, generalconf):
+        """
+        :param clusters_conf: Proxmox configurations
+        :param generalconf : General configuration
+        """
+        self.generalconf = generalconf
+        self.clusters_conf = clusters_conf
+
+        """ LOAD MONGODB """
+        self.mongo = MongoDB(generalconf["mongodb"]["ip"])
+        self.mongo.client = self.mongo.connect()
+        self.mongo.db = self.mongo.client.db
+
+    def run(self):
+        insert_time = time.time()
+
+        self.mongo.insert_datekey(insert_time, 'running')
+
+        for cluster in self.clusters_conf:
+
+
+            datamongo=cluster["user"].replace("\\\\x", "x")
+            convert_to_byte=datamongo.encode("utf-8")
+            print(convert_to_byte)
+            print(pdecrypt(datamongo.encode("utf-8"), self.generalconf["keys"]["key_pvt"]))
+
+
+            """ AUTH """
+            proxmox = Proxmox("Analyse")
+            proxmox.get_ticket("{0}:{1}".format(cluster["url"],
+                                                int(cluster["port"])),
+                               pdecrypt(cluster["user"], self.generalconf["keys"]["key_pvt"]),
+                               pdecrypt(cluster["password"], self.generalconf["keys"]["key_pvt"]))
+
+            """ Get excluded nodes """
+            exclude_nodes = cluster["exclude_nodes"]
+
+            """ UPDATE NODES LIST """
+            nodes_list = proxmox.get_nodes("{0}:{1}".format(cluster["url"], int(cluster["port"])))["value"]
+
+            for value_nodes_list in nodes_list["data"]:
+                if value_nodes_list["node"] not in exclude_nodes:
+                    """ TOTAL COUNT CPU and RAM allocate"""
+                    list_instances = proxmox.get_instance("{0}:{1}".format(cluster["url"], int(cluster["port"])),
+                                                          value_nodes_list["node"], "lxc")["value"]
+
+                    totalcpu = 0
+                    totalram = 0
+                    for key_list_instances, value_list_instances in list_instances.items():
+                        for instances in value_list_instances:
+                            totalcpu = totalcpu + instances["cpus"]
+                            totalram = totalram + instances["maxmem"]
+
+                    value_nodes_list["totalalloccpu"] = totalcpu
+                    value_nodes_list["totalallocram"] = totalram
+                    value_nodes_list["vmcount"] = len(list_instances.items())
+
+                    percent_cpu_alloc = (totalcpu / value_nodes_list["maxcpu"]) * 100
+                    percent_ram_alloc = (totalram / value_nodes_list["mem"]) * 100
+
+                    """
+                    weight of node =
+                    (((Percent Alloc CPU x coef) + ( Percent Alloc RAM x coef)) / Total coef ) * Cluster weight
+                    """
+                    weight = (((percent_cpu_alloc * 2) + (percent_ram_alloc * 4)) / 6) * int(cluster["weight"])
+
+                    value_nodes_list["weight"] = int(weight)
+                    value_nodes_list["date"] = int(insert_time)
+                    value_nodes_list["cluster"] = cluster["name"]
+
+                    self.mongo.insert_node(value_nodes_list)
+
+        self.mongo.update_datekey(int(insert_time), "OK")
+
+        return
+
+    def set_attribution(self, count):
+        """ RETURN cluster and node"""
+        # Search the last valid key
+        lastkeyvalid = self.mongo.get_last_datekey()
+
+        # Get nodes weight
+        nodes_availables = self.mongo.get_nodes_informations(int(lastkeyvalid["value"]))
+
+        if len(nodes_availables) > 1:
+            # Select node name with weight
+            nodes_values = {}
+            for nodes in nodes_availables:
+                nodes_values[nodes["node"]] = nodes["weight"]
+
+            # Sort node by weight
+            sorted_nodes = sorted(nodes_values.items(), key=operator.itemgetter(1))
+
+            slot_distributions = []
+            sorted_nodes_name = []
+
+            # Divise dict sorted_node to two list [name1, name2] [value1, value2]
+            for nodes_sort in sorted_nodes:
+                slot_distributions.append(nodes_sort[1])
+                sorted_nodes_name.append((nodes_sort[0]))
+
+            # Calcul weight on a range of value  0-1 and convert in percent
+            slot_distributions_p = []
+            for s in slot_distributions:
+                slot_distributions_p.append(100-(s/sum(slot_distributions)*100)/100)
+
+            # Generate default list
+            tokens_in_slots = [0] * len(nodes_availables)
+
+            # use distribution algorithm to allocate
+            distrib_final = distribution(int(count), tokens_in_slots, slot_distributions_p)
+
+            # Regenerate final dict !!
+            final = {k: int(v) for k, v in zip(sorted_nodes_name, distrib_final)}
+
+        else:
+            final = {nodes_availables[0]['node']: count}
+
+        return final

code/scripts/main/core/modules/mod_database.py

@@ -0,0 +1,179 @@
+from pymongo import MongoClient
+from bson.json_util import dumps
+import json
+import redis
+import time
+
+class Redis_instance_queue:
+    def __init__(self, server="127.0.0.1", port=6379, db=3, password=None):
+        self.server = server
+        self.port = port
+        self.r = None
+        self.db = db
+        self.password = password
+
+    def connect(self):
+        try:
+            conn = self.r = redis.Redis(
+                host=self.server, port=self.port, db=self.db, password=self.password,
+                charset="utf-8", decode_responses=True)
+            self.r.client_list()
+        except BaseException as err:
+            print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err))
+            conn = False
+        return conn
+
+    def insert_instance_queue(self,  logtext, expir=3000):
+        self.r.set(time.time(), logtext, expir)
+
+
+class Redis_logger:
+    def __init__(self, server="127.0.0.1", port=6379, db=2, password=None):
+        self.server = server
+        self.port = port
+        self.r = None
+        self.db = db
+        self.password = password
+
+    def connect(self):
+        try:
+            conn = self.r = redis.Redis(
+                host=self.server, port=self.port, db=self.db, password=self.password,
+                charset="utf-8", decode_responses=True)
+            self.r.client_list()
+        except BaseException as err:
+            print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err))
+            conn = False
+        return conn
+
+    def insert_logs(self,  logtext, expir=86400*4):
+        self.r.set(time.time(), logtext, expir)
+
+
+class Redis_messages:
+    def __init__(self, server="127.0.0.1", port=6379, db=1, password=None):
+        self.server = server
+        self.port = port
+        self.r = None
+        self.db = db
+        self.password = password
+
+    def connect(self):
+        try:
+            conn = self.r = redis.Redis(
+                host=self.server, port=self.port, db=self.db, password=self.password,
+                charset="utf-8", decode_responses=True)
+            self.r.client_list()
+        except BaseException as err:
+            print("Redis connexion error on {0}:{1} ({2})".format(self.server, self.port, err))
+            conn = False
+        return conn
+
+    def insert_message(self, key, value, expir=86400):
+        self.r.set(key, value, expir)
+
+    def get_message(self, key):
+        return self.r.get(key)
+
+
+class MongoDB:
+    def __init__(self, server="127.0.0.1", port=27017):
+        """
+        :param server:
+        :param port:
+        """
+        self.server = server
+        self.port = port
+        self.collection_system = "system"
+        self.collection_instance = "instances"
+        self.collection_nodes = "nodes"
+        self.collection_clusters = "clusters"
+        self.collection_datekey = "dates"
+        self.port = port
+        self.db = None
+        self.client = None
+
+    def connect(self):
+        try:
+            conn = MongoClient(self.server + ':' + str(self.port))
+            conn.server_info()
+        except BaseException as err:
+            print("MongoDB connexion error on {0}:{1} ({2})".format(self.server, self.port, err))
+            conn = False
+        return conn
+
+    def authenticate(self, user=None, password=None, mechanism='SCRAM-SHA-1'):
+        try:
+            self.client.db.authenticate(user, password, mechanism)
+        except (TypeError, ValueError) as e:
+            raise("MongoDB authentification error on {0}:{1} ({2})".format(self.server, self.port, e))
+
+    """ CLUSTER """
+    def get_clusters_conf(self, cluster=None):
+        if cluster:
+            return json.loads(dumps(self.db[self.collection_clusters].find_one({"name": cluster})))
+        else:
+            return json.loads(dumps(self.db[self.collection_clusters].find({})))
+
+    def insert_new_cluster(self, data):
+        return self.db[self.collection_clusters].insert(data)
+
+    def update_cluster(self, cluster, data):
+        return self.db[self.collection_clusters].update({"vmid": str(cluster)}, {'$set': data}, upsert=False)
+
+    def delete_cluster(self, cluster):
+        return self.db[self.collection_clusters].remove({"cluster": str(cluster)})
+
+
+    """ SYSTEM """
+    def get_system_info(self):
+        return self.db[self.collection_system].find_one({"_id": "0"})
+
+    def update_system_instance_id(self, value):
+        self.db[self.collection_system].update({'_id': "0"}, {'$set': {'instances_number': value}})
+
+    def update_system_instance_ip(self, value):
+        self.db[self.collection_system].update({'_id': "0"}, {'$set': {'IP_current': value}})
+
+    def update_system_free_ip(self, value):
+        self.db[self.collection_system].update({'_id': "0"}, {'$push': {'IP_free': value}}, upsert=False)
+
+    def update_system_delete_ip(self, value):
+        self.db[self.collection_system].update({'_id': "0"}, {'$pull': {'IP_free': value}}, upsert=False)
+
+    """ NODES MANAGEMENT"""
+    def insert_node(self, data):
+        return self.db[self.collection_nodes].insert(data)
+
+    def get_nodes_informations(self, time, node=None):
+        if node:
+            return json.loads(dumps(self.db[self.collection_nodes].find_one({'$and': [{'node': node, 'date': time}]})))
+        else:
+            return json.loads(dumps(self.db[self.collection_nodes].find({'date': time})))
+
+    """ KEY DATE MANAGEMENT"""
+    def insert_datekey(self, date, status):
+        return self.db[self.collection_datekey].insert({'date': int(date), 'status': status})
+
+    def update_datekey(self, date, status):
+        self.db[self.collection_datekey].update({'date': date}, {'$set': {'status': status}}, upsert=False)
+
+    def get_last_datekey(self):
+        last_id = self.db[self.collection_datekey].find({'status': 'OK'}).sort("date", -1).limit(1)
+        return {"value": int(json.loads(dumps(last_id))[0]['date'])}
+
+    """ INSTANCE MANAGEMENT"""
+    def insert_instance(self, data):
+        return self.db[self.collection_instance].insert(data)
+
+    def update_instance(self, vmid, data):
+        self.db[self.collection_instance].update({"vmid": int(vmid)}, {'$set': data}, upsert=False)
+
+    def delete_instance(self, vmid):
+        self.db[self.collection_instance].remove({"vmid": int(vmid)})
+
+    def get_instance(self, vmid):
+        try:
+            return json.loads(dumps(self.db[self.collection_instance].find_one({"vmid": int(vmid)})))
+        except BaseException as serr:
+            raise ("MongoDB error on {0}:{1} ({2})".format(self.server, self.port, serr))

code/scripts/main/core/modules/mod_proxmox.py

@@ -0,0 +1,415 @@
+import requests
+import time
+"""
+Proxmox management API WRAPPER
+"""
+
+
+class NetworkError(RuntimeError):
+    pass
+
+
+class Proxmox:
+    def __init__(self, name):
+        """
+        :param name: Cluster Proxmox
+        """
+        self.name = name
+        self.socket = None
+        self.ticket = None
+        self.PVEAuthCookie = None
+        self.csrf = None
+        self.nodes = None
+        self.status = None
+        self.storage = None
+        self.disks = None
+        self.qemu = None
+        self.config = None
+
+    def get_ticket(self, url, user, password):
+        """
+        Get a new ticket from Proxmox api
+        :param url: Generic ticket url
+        :param user: Proxmox user API
+        :param password: Proxmox password user API
+        """
+        request = "https://{0}/api2/json/access/ticket".format(url)
+        try:
+            self.socket = requests.session()
+            params = {'username': user, 'password': password}
+            self.ticket = self.socket.post(request,  params=params, verify=False, timeout=5)
+
+            if self.ticket.status_code == 200:
+                result = {"result": "OK",
+                          "value": self.ticket.json()
+                          }
+                self.PVEAuthCookie = {'PVEAuthCookie': self.ticket.json()['data']['ticket']}
+                self.csrf = {'CSRFPreventionToken': self.ticket.json()['data']['CSRFPreventionToken']}
+            else:
+                result = {"result": "ERROR",
+                          "target": "{0}".format(url),
+                          "type": "PROXMOX - STATUS CODE",
+                          "customerror": "Error nodes informations. Bad HTTP Status code : "
+                                         "{0} -- {1}".format(self.ticket.status_code, self.ticket.text)
+                          }
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR",
+                      "target": "{0}".format(url),
+                      "type": "PYTHON",
+                      "customerror": "Cannot get ticket session {0} ({1})".format(url, e)}
+
+        return result
+
+    def get_nodes(self, url):
+        """
+        Get Nodes from cluster
+        :param url: Generic node url (node = physical hypervisor)
+        """
+        request = "https://{0}/api2/json/nodes".format(url)
+        try:
+            nodes = self.nodes = self.socket.get(request,
+                                                 cookies=self.PVEAuthCookie,
+                                                 verify=False, timeout=5)
+
+            if nodes.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": nodes.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(url),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error nodes informations. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(nodes.status_code, nodes.text)
+                }
+
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {
+                "result": "ERROR",
+                "target": "{0}".format(url),
+                "type": "PYTHON",
+                "customerror": "Cannot get node information for {0} ({1})".format(url, e)
+            }
+
+        return result
+
+    def get_status(self, url, nodename):
+        """
+        Get node informations
+        :param url: Generic node url
+        :param nodename: Node name (not int id)
+        """
+        request = "https://{0}/api2/json/nodes/{1}/status".format(url, nodename)
+        try:
+            self.status = self.socket.get(request, cookies=self.PVEAuthCookie, verify=False, timeout=5).json()
+            result = {"result": "OK"}
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR", "target": "[{3}]",
+                      "customerror": "Cannot get node information for {0} ({1})".format(url, e, nodename)}
+
+        return result
+
+    def get_storages(self, url, nodename):
+        """
+        Get Storage from nodes
+        :param url: Generic storage url
+        :param nodename: Node name (no int id)
+        """
+        request = "https://{0}/api2/json/nodes/{1}/storage".format(url, nodename)
+        try:
+            self.storage = self.socket.get(request,  cookies=self.PVEAuthCookie, verify=False, timeout=5).json()
+            result = {"result": "OK"}
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR", "target": "[{3}]",
+                      "customerror": "Cannot get storage information for {0} ({1})".format(url, e, nodename)}
+
+        return result
+
+    def get_disks(self, url, nodename, sto_id):
+        """
+        Get VMs disk from storages
+        :param url: Generic content url
+        :param nodename: Node name (no int id)
+        :param sto_id: Storage name (no int id)
+        """
+        request = "https://{0}/api2/json/nodes/{1}/storage/{2}/content".format(url, nodename, sto_id)
+        try:
+            self.disks = self.socket.get(request,  cookies=self.PVEAuthCookie, verify=False, timeout=5).json()
+            result = {"result": "OK"}
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR", "target": "[{3}]",
+                      "customerror": "Cannot get disks information for {0} ({1})".format(url, e, nodename)}
+
+        return result
+
+    def get_instance(self, url, nodename, category):
+        """
+        Get basic VMs informations from nodes
+        :param url: Generic qemu url
+        :param nodename: Node name (not int id)
+        :param category: lxc or qemu
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}".format(url, nodename, category)
+        try:
+            instances = self.socket.get(request,
+                                        cookies=self.PVEAuthCookie,
+                                        verify=False, timeout=5)
+
+            if instances.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": instances.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(url),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error nodes informations. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(instances.status_code, instances.text)
+                }
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {
+                "result": "ERROR",
+                "target": "{0}".format(url),
+                "type": "PYTHON",
+                "customerror": "Cannot get VM information for {0} {1} ({2})".format(url, nodename, e)
+            }
+
+        return result
+
+    def get_config(self, url, nodename, category, instanceid):
+        """
+        Get avanced VM information from nodes
+        :param url: Generic qemu config url
+        :param category: lxc or qemu
+        :param nodename: Node name (not int id)
+        :param instanceid: VM id (int id)
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}/{3}/config".format(url, nodename, category, instanceid)
+        try:
+            config = self.socket.get(request,  cookies=self.PVEAuthCookie, verify=False, timeout=5)
+            if config.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": config.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(url),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error nodes informations. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(config.status_code, config.text)
+                }
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(url),
+                    "type": "PYTHON",
+                    "customerror": "Cannot get VM information for {0} {1} ({2})".format(url, nodename, e)
+                }
+
+        return result
+
+    def create_instance(self, url, nodename, category, data):
+        """
+        :param url: 
+        :param nodename: 
+        :param category: 
+        :param data: 
+        :return: 
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}".format(url, nodename, category)
+        try:
+            createvm = self.socket.post(request,
+                                        data=data,
+                                        headers=self.csrf,
+                                        cookies=self.PVEAuthCookie,
+                                        verify=False,
+                                        timeout=5)
+
+            if createvm.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": createvm.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(nodename),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error creating Container. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(createvm.status_code, createvm.text)
+                }
+
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR",
+                      "target": "{0}".format(nodename),
+                      "type": "PYTHON",
+                      "customerror": "Cannot create this instance on {0} : ({1})".format(url, e)}
+
+        return result
+
+    def delete_instance(self, url, nodename, category, vmid):
+        """
+        :param url: 
+        :param nodename: 
+        :param category: 
+        :param vmid:
+        :return: 
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}/{3}".format(url, nodename, category, vmid)
+        try:
+            deletevm = self.socket.delete(request,
+                                          headers=self.csrf,
+                                          cookies=self.PVEAuthCookie,
+                                          verify=False,
+                                          timeout=5
+                                          )
+
+            if deletevm.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": deletevm.json()
+                }
+
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(nodename),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error delete Container. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(deletevm.status_code, deletevm.text)
+                }
+
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR",
+                      "target": "{0}".format(nodename),
+                      "type": "PYTHON",
+                      "customerror": "Cannot delete Container ({2}) on {0} : ({1})".format(url, e, vmid)}
+
+        return result
+
+    def status_instance(self, url, nodename, category, vmid, action):
+        """
+        :param url: 
+        :param nodename: 
+        :param category: 
+        :param vmid:
+        :param action:
+        :return: 
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}/{3}/status/{4}".format(url, nodename, category, vmid, action)
+        try:
+            if action == "current":
+                statusm = self.socket.get(request,
+                                          cookies=self.PVEAuthCookie,
+                                          verify=False,
+                                          timeout=5)
+            else:
+                statusm = self.socket.post(request,
+                                           headers=self.csrf,
+                                           cookies=self.PVEAuthCookie,
+                                           verify=False,
+                                           timeout=5)
+
+            if statusm.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": statusm.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(nodename),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error action Container. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(statusm.status_code, statusm.text)
+                }
+
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {"result": "ERROR",
+                      "target": "{0}".format(nodename),
+                      "type": "PYTHON",
+                      "customerror": "Cannot do this action this instance ({2}) on {0} : ({1})".format(url, e, vmid)}
+
+        return result
+
+    def resize_instance(self, url, nodename, category, instanceid, data):
+        """
+        :param url: 
+        :param nodename: 
+        :param category: 
+        :param instanceid: 
+        :param data: 
+        :return: 
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}/{3}/config".format(url, nodename, category, instanceid)
+        try:
+            resizevm = self.socket.put(request,
+                                       data=data,
+                                       headers=self.csrf,
+                                       cookies=self.PVEAuthCookie,
+                                       verify=False,
+                                       timeout=5)
+
+            if resizevm.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": resizevm.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(nodename),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error resizing container. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(resizevm.status_code, resizevm.text)
+                }
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(nodename),
+                    "type": "PYTHON",
+                    "customerror": "Cannot resize this instance {2} on {0} : ({1})".format(url, e, instanceid)
+                }
+
+        return result
+
+    def stats_instance(self, url, nodename, category, instanceid):
+        """
+        :param url: 
+        :param nodename: 
+        :param category: 
+        :param instanceid: 
+        :return: 
+        """
+        request = "https://{0}/api2/json/nodes/{1}/{2}/{3}/rrddata".format(url, nodename, category, instanceid)
+        try:
+            statsvm = self.socket.get(request, cookies=self.PVEAuthCookie, verify=False, timeout=5)
+
+            if statsvm.status_code == 200:
+                result = {
+                    "result": "OK",
+                    "value": statsvm.json()
+                }
+            else:
+                result = {
+                    "result": "ERROR",
+                    "target": "{0}".format(nodename),
+                    "type": "PROXMOX - STATUS CODE",
+                    "customerror": "Error to find statistic for instance {2}. Bad HTTP Status code : "
+                                   "{0} -- {1}".format(statsvm.status_code, statsvm.text, instanceid)
+                }
+        except (TypeError, ValueError, requests.exceptions.RequestException) as e:
+            result = {
+                "result": "ERROR",
+                "target": "{0}".format(nodename),
+                "type": "PYTHON",
+                "customerror": "Cannot find statistic for this instance {2} on {0} : ({1})".format(url, e, instanceid)
+            }
+
+        return result

code/scripts/main/core/modules/mod_queue.py

@@ -0,0 +1,20 @@
+"""
+Queue Management
+"""
+
+class Queue():
+    def __init__(self):
+        self.name = "command"
+
+    def insert_command_queue(self):
+
+        return
+
+    def delete_command(self):
+        return
+
+    def serve_command(self):
+        return
+
+    def queue_stats(self):
+        return

code/scripts/main/core/modules/mod_vhost.py

@@ -0,0 +1,34 @@
+import requests
+
+"""
+VHOST Management
+"""
+
+class Nginx:
+    def __init__(self, clientid):
+        """
+        :param clientid: 
+        """
+
+class Apache:
+    def __init__(self, siteid):
+        """
+        :param name: Cluster Proxmox
+        """
+        self.siteid = siteid
+
+    def get_vhost(self):
+        """
+        List all vhost available
+        :return: 
+        """
+        return
+
+    def create_vhost(self):
+        return
+
+    def delete_vhost(self):
+        return
+
+    def update_vhost(self):
+        return