overhaul of rtp view, added simple caching, added rrd graphs

.gitignore

@@ -1,2 +1,3 @@
 config.py
 __pycache__/*
+rrd/*

app.py

@@ -5,32 +5,47 @@ import subprocess
 from db import *
 from starrs import *
 from proxmox import *
+from werkzeug.contrib.cache import SimpleCache
 from flask_pyoidc.flask_pyoidc import OIDCAuthentication
 from flask import Flask, render_template, request, redirect, send_from_directory, session
 
 app = Flask(__name__)
-
 config = os.path.join(app.config.get('ROOT_DIR', os.getcwd()), "config.py")
-
 app.config.from_pyfile(config)
-
 app.config["GIT_REVISION"] = subprocess.check_output(
     ['git', 'rev-parse', '--short', 'HEAD']).decode('utf-8').rstrip()
-
 auth = OIDCAuthentication(
     app,
     issuer=app.config['OIDC_ISSUER'],
     client_registration_info=app.config['OIDC_CLIENT_CONFIG'])
+cache = SimpleCache()
 
 
 @app.route("/")
+@app.route("/user/<string:user>")
 @auth.oidc_auth
-def list_vms():
+def list_vms(user=None):
-    user = session['userinfo']['preferred_username']
+    rtp_view = False
     rtp = 'rtp' in session['userinfo']['groups']
     proxmox = connect_proxmox()
-    vms = get_vms_for_user(proxmox, user, rtp)
+    if user and not rtp:
-    return render_template('list_vms.html', username=user, rtp=rtp, vms=vms)
+        return '', 403
+    elif user and rtp:
+        vms = get_vms_for_user(proxmox, user)
+        rtp_view = user
+        user = session['userinfo']['preferred_username']
+    elif rtp:
+        user = session['userinfo']['preferred_username']
+        vms = cache.get('vms')
+        if vms is None:
+            vms = get_vms_for_rtp(proxmox)
+            cache.set('vms', vms, timeout=5 * 60)
+        rtp_view = True
+    else:
+        user = session['userinfo']['preferred_username']
+        vms = get_vms_for_user(proxmox, user)
+    return render_template(
+        'list_vms.html', username=user, rtp=rtp, rtp_view=rtp_view, vms=vms)
 
 
 @app.route("/isos")
@@ -227,7 +242,7 @@ def create():
     if request.method == 'GET':
         usage = get_user_usage(proxmox, user)
         limits = get_user_usage_limits(user)
-        percents = get_user_usage_percent(proxmox, usage, limits)
+        percents = get_user_usage_percent(proxmox, user, usage, limits)
         isos = get_isos(proxmox, app.config['PROXMOX_ISO_STORAGE'])
         return render_template(
             'create.html',
@@ -304,6 +319,12 @@ def limits():
         return '', 403
 
 
+@app.route('/vm/<string:vmid>/rrd/<path:path>')
+@auth.oidc_auth
+def send_rrd(vmid, path):
+    return send_from_directory("rrd/{}".format(vmid), path)
+
+
 @app.route('/novnc/<path:path>')
 @auth.oidc_auth
 def send_novnc(path):

cron.py

@@ -1,19 +1,47 @@
+import os
 from db import *
-from config import *
 from starrs import *
 from proxmox import *
+from flask import Flask, current_app
+
+app = Flask(__name__)
+config = os.path.join(app.config.get('ROOT_DIR', os.getcwd()), "config.py")
+app.config.from_pyfile(config)
 
 
 def process_expired_vms():
-    proxmox = connect_proxmox(PROXMOX_HOST, PROXMOX_USER, PROXMOX_PASS)
+    proxmox = connect_proxmox()
-    starrs = connect_starrs(STARRS_DB_NAME, STARRS_DB_USER, STARRS_DB_HOST,
+    starrs = connect_starrs()
-                            STARRS_DB_PASS)
     expired_vms = get_expired_vms()
-    for vmid in expired_vms:
+    print(expired_vms)
-        vmname = get_vm_config(proxmox, vmid)['name']
-        delete_vm(proxmox, starrs, vmid)
-        delete_starrs(starrs, vmname)
-        delete_vm_expire(vmid)
 
 
-process_expired_vms()
+#    for vmid in expired_vms:
+#        vmname = get_vm_config(proxmox, vmid)['name']
+#        delete_vm(proxmox, starrs, vmid)
+#        delete_starrs(starrs, vmname)
+#        delete_vm_expire(vmid)
+
+
+def get_rrd_graphs():
+    proxmox = connect_proxmox()
+    pools = get_pools(proxmox)
+    for pool in pools:
+        vms = proxmox.pools(pool).get()['members']
+        for vm in vms:
+            vm_dir = "rrd/{}".format(vm['vmid'])
+            if not os.path.exists(vm_dir):
+                os.makedirs(vm_dir)
+            sources = [
+                'cpu', 'mem', 'netin', 'netout', 'diskread', 'diskwrite'
+            ]
+            for source in sources:
+                image = get_rrd_for_vm(proxmox, vm['vmid'], source, 'day')
+                with open("rrd/{}/{}.png".format(vm['vmid'], source),
+                          'wb') as f:
+                    f.write(image.encode('raw_unicode_escape'))
+
+
+with app.app_context():
+    process_expired_vms()
+    get_rrd_graphs()

proxmox.py

@@ -1,40 +1,47 @@
 import time
-from flask import current_app as app
+from functools import lru_cache
 from proxmoxer import ProxmoxAPI
+from flask import current_app as app
 from db import *
 
 
 def connect_proxmox():
     try:
         proxmox = ProxmoxAPI(
-            app.config['PROXMOX_HOST'], user=app.config['PROXMOX_USER'], password=app.config['PROXMOX_PASS'], verify_ssl=False)
+            app.config['PROXMOX_HOST'],
+            user=app.config['PROXMOX_USER'],
+            password=app.config['PROXMOX_PASS'],
+            verify_ssl=False)
     except:
         print("Unable to connect to Proxmox!")
         raise
     return proxmox
 
 
-def get_vms_for_user(proxmox, user, rtp=False):
+def get_vms_for_user(proxmox, user):
     pools = get_pools(proxmox)
-    if not rtp:
+    if user not in pools:
-        if user not in pools:
+        proxmox.pools.post(poolid=user, comment='Managed by Proxstar')
-            proxmox.pools.post(poolid=user, comment='Managed by Proxstar')
+    vms = proxmox.pools(user).get()['members']
-        vms = proxmox.pools(user).get()['members']
+    for vm in vms:
-        for vm in vms:
+        if 'name' not in vm:
-            if 'name' not in vm:
+            vms.remove(vm)
-                vms.remove(vm)
+    vms = sorted(vms, key=lambda k: k['name'])
-        vms = sorted(vms, key=lambda k: k['name'])
+    return vms
-        return vms
+
-    else:
+
-        pool_vms = []
+def get_vms_for_rtp(proxmox):
-        for pool in pools:
+    pools = get_pools(proxmox)
-            vms = proxmox.pools(pool).get()['members']
+    pool_vms = []
-            for vm in vms:
+    for pool in pools:
-                if 'name' not in vm:
+        pool_dict = dict()
-                    vms.remove(vm)
+        pool_dict['user'] = pool
-            vms = sorted(vms, key=lambda k: k['name'])
+        pool_dict['usage'] = get_user_usage(proxmox, pool)
-            pool_vms.append([pool, vms])
+        pool_dict['limits'] = get_user_usage_limits(pool)
-        return pool_vms
+        pool_dict['percents'] = get_user_usage_percent(
+            proxmox, pool, pool_dict['usage'], pool_dict['limits'])
+        pool_vms.append(pool_dict)
+    return pool_vms
 
 
 def get_user_allowed_vms(proxmox, user):
@@ -102,10 +109,9 @@ def get_vm_disk_size(proxmox, vmid, config=None, name='virtio0'):
     if not config:
         config = get_vm_config(proxmox, vmid)
     disk_size = config[name].split(',')
-    if 'size' in disk_size[0]:
+    for split in disk_size:
-        disk_size = disk_size[0].split('=')[1].rstrip('G')
+        if 'size' in split:
-    else:
+            disk_size = split.split('=')[1].rstrip('G')
-        disk_size = disk_size[1].split('=')[1].rstrip('G')
     return disk_size
 
 
@@ -118,10 +124,9 @@ def get_vm_disks(proxmox, vmid, config=None):
         if any(disk_type in key for disk_type in valid_disk_types):
             if 'scsihw' not in key and 'cdrom' not in val:
                 disk_size = val.split(',')
-                if 'size' in disk_size[0]:
+                for split in disk_size:
-                    disk_size = disk_size[0].split('=')[1].rstrip('G')
+                    if 'size' in split:
-                else:
+                        disk_size = split.split('=')[1].rstrip('G')
-                    disk_size = disk_size[1].split('=')[1].rstrip('G')
                 disks.append([key, disk_size])
     disks = sorted(disks, key=lambda x: x[0])
     return disks
@@ -151,7 +156,7 @@ def get_user_usage(proxmox, user):
         if 'status' in vm:
             if vm['status'] == 'running' or vm['status'] == 'paused':
                 usage['cpu'] += int(config['cores'] * config.get('sockets', 1))
-                usage['mem'] += (int(config['memory']) // 1024)
+                usage['mem'] += (int(config['memory']) / 1024)
             for disk in get_vm_disks(proxmox, vm['vmid'], config):
                 usage['disk'] += int(disk[1])
     return usage
@@ -168,15 +173,18 @@ def check_user_usage(proxmox, user, vm_cpu, vm_mem, vm_disk):
         return 'exceeds_disk_limit'
 
 
-def get_user_usage_percent(proxmox, usage=None, limits=None):
+def get_user_usage_percent(proxmox, user, usage=None, limits=None):
     percents = dict()
     if not usage:
         usage = get_user_usage(proxmox, user)
     if not limits:
         limits = get_user_usage_limits(user)
-    percents['cpu'] = round(int(usage['cpu']) / int(limits['cpu']) * 100)
+    percents['cpu'] = round(usage['cpu'] / limits['cpu'] * 100)
-    percents['mem'] = round(int(usage['mem']) / int(limits['mem']) * 100)
+    percents['mem'] = round(usage['mem'] / limits['mem'] * 100)
-    percents['disk'] = round(int(usage['disk']) / int(limits['disk']) * 100)
+    percents['disk'] = round(usage['disk'] / limits['disk'] * 100)
+    for resource in percents:
+        if percents[resource] > 100:
+            percents[resource] = 100
     return percents
 
 
@@ -255,3 +263,9 @@ def get_pools(proxmox):
             pools.append(poolid)
     pools = sorted(pools)
     return pools
+
+
+def get_rrd_for_vm(proxmox, vmid, source, time):
+    node = proxmox.nodes(get_vm_node(proxmox, vmid))
+    image = node.qemu(vmid).rrd.get(ds=source, timeframe=time)['image']
+    return image

starrs.py

@@ -6,7 +6,8 @@ def connect_starrs():
     try:
         starrs = psycopg2.connect(
             "dbname='{}' user='{}' host='{}' password='{}'".format(
-                app.config['STARRS_DB_NAME'], app.config['STARRS_DB_USER'], app.config['STARRS_DB_HOST'], app.config['STARRS_DB_PASS']))
+                app.config['STARRS_DB_NAME'], app.config['STARRS_DB_USER'],
+                app.config['STARRS_DB_HOST'], app.config['STARRS_DB_PASS']))
     except:
         print("Unable to connect to STARRS database.")
         raise

static/css/styles.css

@@ -105,6 +105,37 @@ table, th, td {
 }
 
 .usage-limit {
-    width: 80%;
     margin: 0px auto;
 }
+
+.rrd-graph {
+    width: 800px;
+    height: 200px;
+    padding-top: 10px;
+    padding-bottom: 10px;
+}
+
+.profile {
+    float: left;
+    width: 35%;
+    height: 60px;
+}
+
+.profile-img {
+    max-width: 100%;
+    max-height: 100%;
+}
+
+.progress {
+    margin-top: 2px;
+    margin-bottom: 0px;
+    float: right;
+    width: 85%;
+    height: 12px;
+}
+
+.resource-bar {
+    width: 60%;
+    float: right;
+    padding-bottom: 2px;
+}

static/js/script.js

@@ -584,18 +584,21 @@ $(".edit-limit").click(function(){
     cpu_text.innerHTML = 'CPU';
     options.append(cpu_text);
     var cpu = document.createElement('input');
+    cpu.type = 'number';
     cpu.defaultValue = cur_cpu;
     options.append(cpu);
     mem_text = document.createElement('p');
     mem_text.innerHTML = 'Memory (GB)';
     options.append(mem_text);
     var mem = document.createElement('input');
+    mem.type = 'number';
     mem.defaultValue = cur_mem;
     options.append(mem)
     disk_text = document.createElement('p');
     disk_text.innerHTML = 'Disk (GB)';
     options.append(disk_text);
     var disk = document.createElement('input');
+    disk.type = 'number';
     disk.defaultValue = cur_disk;
     options.append(disk)
     swal({

templates/create.html

@@ -27,6 +27,7 @@
                     <div class="form-group">
                         <label for="mem">Memory</label>
                         <select name="mem" id="mem" class="form-control">
+                            <option value="512">512MB</option>
                             {% for i in range(1, limits['mem'] + 1) %}
                             <option value="{{ i  * 1024 }}">{{ i }}GB</option>
                             {% endfor %}

templates/list_vms.html

@@ -3,6 +3,15 @@
 
 <div class="container">
     <div class="row">
+        {% if rtp and rtp_view != True %}
+        <div class="col-md-12 col-sm-12">
+            <div class="panel panel-default">
+                <div class="panel-heading">
+                    <p>{{ rtp_view }}</p>
+                </div>
+            </div>
+        </div>
+        {% endif %}
         {% if not vms %}
         <div class="col-md-12 col-sm-12">
             <div class="panel panel-default">
@@ -11,7 +20,7 @@
                 </div>
             </div>
         </div>
-        {% elif not rtp %}
+        {% elif rtp_view != True %}
             {% for vm in vms %}
                 <div class="col-md-3 col-sm-4 col-xs-6">
                     <div class="panel panel-default">
@@ -26,29 +35,58 @@
             {% endfor %}
         {% else %}
             {% for pool in vms %}
-                {% if pool[1] %}
+                <div class="col-md-3 col-sm-4 col-xs-6">
-                <div class="col-md-12 col-sm-12">
                     <div class="panel panel-default">
                         <div class="panel-heading">
-                            <h3 class="panel-title">{{ pool[0] }}</h3>
+                            <h3 class="panel-title">
+                                <a href="/user/{{ pool['user'] }}">
+                                    <p>{{ pool['user'] }}</p>
+                                </a>
+                            </h3>
                         </div>
                         <div class="panel-body">
-                            {% for vm in pool[1] %}
+                            <div class="profile">
-                                <div class="col-md-3 col-sm-4 col-xs-6">
+                                <img class="profile-img" src="https://profiles.csh.rit.edu/image/{{ pool['user'] }}" title="{{ pool['user'] }}">
-                                    <div class="panel panel-default">
+                            </div>
-                                        <div class="panel-body">
+                            <div class="resource-bar">
-                                            <a href="/vm/{{ vm['vmid'] }}">
+                                <span class="glyphicon glyphicon-cog pull-left"></span>
-                                                <p>{{ vm['name'] }}</p>
+                                <div class="progress">
-                                            </a>
+                                    {% if pool['percents']['cpu'] <= 50 %}
-                                            <p>Status: {{ vm['status'] }}</p>
+                                    <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="{{ pool['percents']['cpu'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['cpu'] }}%"></div>
-                                        </div>
+                                    {% elif pool['percents']['cpu'] <= 75 %}
-                                    </div>
+                                    <div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="{{ pool['percents']['cpu'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['cpu'] }}%"></div>
+                                    {% else %}
+                                    <div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="{{ pool['percents']['cpu'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['cpu'] }}%"></div>
+                                    {% endif %}
                                 </div>
-                            {% endfor %}
+                            </div>
+                            <div class="resource-bar">
+                                <span class="glyphicon glyphicon-tasks pull-left"></span>
+                                <div class="progress">
+                                    {% if pool['percents']['mem'] <= 50 %}
+                                    <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="{{ pool['percents']['mem'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['mem'] }}%"></div>
+                                    {% elif pool['percents']['mem'] <= 75 %}
+                                    <div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="{{ pool['percents']['mem'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['mem'] }}%"></div>
+                                    {% else %}
+                                    <div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="{{ pool['percents']['mem'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['mem'] }}%"></div>
+                                    {% endif %}
+                                </div>
+                            </div>
+                            <div class="resource-bar">
+                                <span class="glyphicon glyphicon-hdd pull-left"></span>
+                                <div class="progress">
+                                    {% if pool['percents']['disk'] <= 50 %}
+                                    <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="{{ pool['percents']['disk'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['disk'] }}%"></div>
+                                    {% elif pool['percents']['disk'] <= 75 %}
+                                    <div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="{{ pool['percents']['disk'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['disk'] }}%"></div>
+                                    {% else %}
+                                    <div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="{{ pool['percents']['disk'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['disk'] }}%"></div>
+                                    {% endif %}
+                                </div>
+                            </div>
                         </div>
                     </div>
                 </div>
-                {% endif %}
             {% endfor %}
         {% endif %}
     </div>

templates/vm_details.html

@@ -96,7 +96,11 @@
                         </dd>
                         <dt>Memory</dt>
                         <dd>
+                            {% if vm['config']['memory'] < 1024 %}
+                            {{ vm['config']['memory'] }}MB
+                            {% else %}
                             {{ vm['config']['memory'] // 1024 }}GB
+                            {% endif %}
                             {% if vm['qmpstatus'] == 'running' or vm['qmpstatus'] == 'paused' %}
                             <button class="btn btn-default proxstar-changebtn" id="change-mem" data-vmid="{{ vm['vmid'] }}" data-mem="{{ vm['config']['memory'] // 1024 }}" data-usage="{{ usage['mem'] - vm['config']['memory'] // 1024 }}" data-limit="{{ limits['mem'] }}">
                                 <span class="glyphicon glyphicon-cog"></span>
@@ -118,6 +122,21 @@
                 </div>
             </div>
         </div>
+        <div class="col-md-12 col-sm-12">
+            <div class="panel panel-default">
+                <div class="panel-heading">
+                    <h3 class="panel-title">RRD Graphs</h3>
+                </div>
+                <div class="panel-body">
+                    <img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/cpu.png">
+                    <img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/mem.png">
+                    <img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/netin.png">
+                    <img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/netout.png">
+                    <img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/diskread.png">
+                    <img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/diskwrite.png">
+                </div>
+            </div>
+        </div>
     </div>
 </div>