overhaul of rtp view, added simple caching, added rrd graphs

This commit is contained in:
Jordan Rodgers 2017-12-13 17:08:57 -05:00
parent b9e5236f0a
commit 2b69443930
10 changed files with 229 additions and 72 deletions

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
config.py config.py
__pycache__/* __pycache__/*
rrd/*

39
app.py
View file

@ -5,32 +5,47 @@ import subprocess
from db import * from db import *
from starrs import * from starrs import *
from proxmox import * from proxmox import *
from werkzeug.contrib.cache import SimpleCache
from flask_pyoidc.flask_pyoidc import OIDCAuthentication from flask_pyoidc.flask_pyoidc import OIDCAuthentication
from flask import Flask, render_template, request, redirect, send_from_directory, session from flask import Flask, render_template, request, redirect, send_from_directory, session
app = Flask(__name__) app = Flask(__name__)
config = os.path.join(app.config.get('ROOT_DIR', os.getcwd()), "config.py") config = os.path.join(app.config.get('ROOT_DIR', os.getcwd()), "config.py")
app.config.from_pyfile(config) app.config.from_pyfile(config)
app.config["GIT_REVISION"] = subprocess.check_output( app.config["GIT_REVISION"] = subprocess.check_output(
['git', 'rev-parse', '--short', 'HEAD']).decode('utf-8').rstrip() ['git', 'rev-parse', '--short', 'HEAD']).decode('utf-8').rstrip()
auth = OIDCAuthentication( auth = OIDCAuthentication(
app, app,
issuer=app.config['OIDC_ISSUER'], issuer=app.config['OIDC_ISSUER'],
client_registration_info=app.config['OIDC_CLIENT_CONFIG']) client_registration_info=app.config['OIDC_CLIENT_CONFIG'])
cache = SimpleCache()
@app.route("/") @app.route("/")
@app.route("/user/<string:user>")
@auth.oidc_auth @auth.oidc_auth
def list_vms(): def list_vms(user=None):
user = session['userinfo']['preferred_username'] rtp_view = False
rtp = 'rtp' in session['userinfo']['groups'] rtp = 'rtp' in session['userinfo']['groups']
proxmox = connect_proxmox() proxmox = connect_proxmox()
vms = get_vms_for_user(proxmox, user, rtp) if user and not rtp:
return render_template('list_vms.html', username=user, rtp=rtp, vms=vms) return '', 403
elif user and rtp:
vms = get_vms_for_user(proxmox, user)
rtp_view = user
user = session['userinfo']['preferred_username']
elif rtp:
user = session['userinfo']['preferred_username']
vms = cache.get('vms')
if vms is None:
vms = get_vms_for_rtp(proxmox)
cache.set('vms', vms, timeout=5 * 60)
rtp_view = True
else:
user = session['userinfo']['preferred_username']
vms = get_vms_for_user(proxmox, user)
return render_template(
'list_vms.html', username=user, rtp=rtp, rtp_view=rtp_view, vms=vms)
@app.route("/isos") @app.route("/isos")
@ -227,7 +242,7 @@ def create():
if request.method == 'GET': if request.method == 'GET':
usage = get_user_usage(proxmox, user) usage = get_user_usage(proxmox, user)
limits = get_user_usage_limits(user) limits = get_user_usage_limits(user)
percents = get_user_usage_percent(proxmox, usage, limits) percents = get_user_usage_percent(proxmox, user, usage, limits)
isos = get_isos(proxmox, app.config['PROXMOX_ISO_STORAGE']) isos = get_isos(proxmox, app.config['PROXMOX_ISO_STORAGE'])
return render_template( return render_template(
'create.html', 'create.html',
@ -304,6 +319,12 @@ def limits():
return '', 403 return '', 403
@app.route('/vm/<string:vmid>/rrd/<path:path>')
@auth.oidc_auth
def send_rrd(vmid, path):
return send_from_directory("rrd/{}".format(vmid), path)
@app.route('/novnc/<path:path>') @app.route('/novnc/<path:path>')
@auth.oidc_auth @auth.oidc_auth
def send_novnc(path): def send_novnc(path):

48
cron.py
View file

@ -1,19 +1,47 @@
import os
from db import * from db import *
from config import *
from starrs import * from starrs import *
from proxmox import * from proxmox import *
from flask import Flask, current_app
app = Flask(__name__)
config = os.path.join(app.config.get('ROOT_DIR', os.getcwd()), "config.py")
app.config.from_pyfile(config)
def process_expired_vms(): def process_expired_vms():
proxmox = connect_proxmox(PROXMOX_HOST, PROXMOX_USER, PROXMOX_PASS) proxmox = connect_proxmox()
starrs = connect_starrs(STARRS_DB_NAME, STARRS_DB_USER, STARRS_DB_HOST, starrs = connect_starrs()
STARRS_DB_PASS)
expired_vms = get_expired_vms() expired_vms = get_expired_vms()
for vmid in expired_vms: print(expired_vms)
vmname = get_vm_config(proxmox, vmid)['name']
delete_vm(proxmox, starrs, vmid)
delete_starrs(starrs, vmname)
delete_vm_expire(vmid)
process_expired_vms() # for vmid in expired_vms:
# vmname = get_vm_config(proxmox, vmid)['name']
# delete_vm(proxmox, starrs, vmid)
# delete_starrs(starrs, vmname)
# delete_vm_expire(vmid)
def get_rrd_graphs():
proxmox = connect_proxmox()
pools = get_pools(proxmox)
for pool in pools:
vms = proxmox.pools(pool).get()['members']
for vm in vms:
vm_dir = "rrd/{}".format(vm['vmid'])
if not os.path.exists(vm_dir):
os.makedirs(vm_dir)
sources = [
'cpu', 'mem', 'netin', 'netout', 'diskread', 'diskwrite'
]
for source in sources:
image = get_rrd_for_vm(proxmox, vm['vmid'], source, 'day')
with open("rrd/{}/{}.png".format(vm['vmid'], source),
'wb') as f:
f.write(image.encode('raw_unicode_escape'))
with app.app_context():
process_expired_vms()
get_rrd_graphs()

View file

@ -1,40 +1,47 @@
import time import time
from flask import current_app as app from functools import lru_cache
from proxmoxer import ProxmoxAPI from proxmoxer import ProxmoxAPI
from flask import current_app as app
from db import * from db import *
def connect_proxmox(): def connect_proxmox():
try: try:
proxmox = ProxmoxAPI( proxmox = ProxmoxAPI(
app.config['PROXMOX_HOST'], user=app.config['PROXMOX_USER'], password=app.config['PROXMOX_PASS'], verify_ssl=False) app.config['PROXMOX_HOST'],
user=app.config['PROXMOX_USER'],
password=app.config['PROXMOX_PASS'],
verify_ssl=False)
except: except:
print("Unable to connect to Proxmox!") print("Unable to connect to Proxmox!")
raise raise
return proxmox return proxmox
def get_vms_for_user(proxmox, user, rtp=False): def get_vms_for_user(proxmox, user):
pools = get_pools(proxmox) pools = get_pools(proxmox)
if not rtp: if user not in pools:
if user not in pools: proxmox.pools.post(poolid=user, comment='Managed by Proxstar')
proxmox.pools.post(poolid=user, comment='Managed by Proxstar') vms = proxmox.pools(user).get()['members']
vms = proxmox.pools(user).get()['members'] for vm in vms:
for vm in vms: if 'name' not in vm:
if 'name' not in vm: vms.remove(vm)
vms.remove(vm) vms = sorted(vms, key=lambda k: k['name'])
vms = sorted(vms, key=lambda k: k['name']) return vms
return vms
else:
pool_vms = [] def get_vms_for_rtp(proxmox):
for pool in pools: pools = get_pools(proxmox)
vms = proxmox.pools(pool).get()['members'] pool_vms = []
for vm in vms: for pool in pools:
if 'name' not in vm: pool_dict = dict()
vms.remove(vm) pool_dict['user'] = pool
vms = sorted(vms, key=lambda k: k['name']) pool_dict['usage'] = get_user_usage(proxmox, pool)
pool_vms.append([pool, vms]) pool_dict['limits'] = get_user_usage_limits(pool)
return pool_vms pool_dict['percents'] = get_user_usage_percent(
proxmox, pool, pool_dict['usage'], pool_dict['limits'])
pool_vms.append(pool_dict)
return pool_vms
def get_user_allowed_vms(proxmox, user): def get_user_allowed_vms(proxmox, user):
@ -102,10 +109,9 @@ def get_vm_disk_size(proxmox, vmid, config=None, name='virtio0'):
if not config: if not config:
config = get_vm_config(proxmox, vmid) config = get_vm_config(proxmox, vmid)
disk_size = config[name].split(',') disk_size = config[name].split(',')
if 'size' in disk_size[0]: for split in disk_size:
disk_size = disk_size[0].split('=')[1].rstrip('G') if 'size' in split:
else: disk_size = split.split('=')[1].rstrip('G')
disk_size = disk_size[1].split('=')[1].rstrip('G')
return disk_size return disk_size
@ -118,10 +124,9 @@ def get_vm_disks(proxmox, vmid, config=None):
if any(disk_type in key for disk_type in valid_disk_types): if any(disk_type in key for disk_type in valid_disk_types):
if 'scsihw' not in key and 'cdrom' not in val: if 'scsihw' not in key and 'cdrom' not in val:
disk_size = val.split(',') disk_size = val.split(',')
if 'size' in disk_size[0]: for split in disk_size:
disk_size = disk_size[0].split('=')[1].rstrip('G') if 'size' in split:
else: disk_size = split.split('=')[1].rstrip('G')
disk_size = disk_size[1].split('=')[1].rstrip('G')
disks.append([key, disk_size]) disks.append([key, disk_size])
disks = sorted(disks, key=lambda x: x[0]) disks = sorted(disks, key=lambda x: x[0])
return disks return disks
@ -151,7 +156,7 @@ def get_user_usage(proxmox, user):
if 'status' in vm: if 'status' in vm:
if vm['status'] == 'running' or vm['status'] == 'paused': if vm['status'] == 'running' or vm['status'] == 'paused':
usage['cpu'] += int(config['cores'] * config.get('sockets', 1)) usage['cpu'] += int(config['cores'] * config.get('sockets', 1))
usage['mem'] += (int(config['memory']) // 1024) usage['mem'] += (int(config['memory']) / 1024)
for disk in get_vm_disks(proxmox, vm['vmid'], config): for disk in get_vm_disks(proxmox, vm['vmid'], config):
usage['disk'] += int(disk[1]) usage['disk'] += int(disk[1])
return usage return usage
@ -168,15 +173,18 @@ def check_user_usage(proxmox, user, vm_cpu, vm_mem, vm_disk):
return 'exceeds_disk_limit' return 'exceeds_disk_limit'
def get_user_usage_percent(proxmox, usage=None, limits=None): def get_user_usage_percent(proxmox, user, usage=None, limits=None):
percents = dict() percents = dict()
if not usage: if not usage:
usage = get_user_usage(proxmox, user) usage = get_user_usage(proxmox, user)
if not limits: if not limits:
limits = get_user_usage_limits(user) limits = get_user_usage_limits(user)
percents['cpu'] = round(int(usage['cpu']) / int(limits['cpu']) * 100) percents['cpu'] = round(usage['cpu'] / limits['cpu'] * 100)
percents['mem'] = round(int(usage['mem']) / int(limits['mem']) * 100) percents['mem'] = round(usage['mem'] / limits['mem'] * 100)
percents['disk'] = round(int(usage['disk']) / int(limits['disk']) * 100) percents['disk'] = round(usage['disk'] / limits['disk'] * 100)
for resource in percents:
if percents[resource] > 100:
percents[resource] = 100
return percents return percents
@ -255,3 +263,9 @@ def get_pools(proxmox):
pools.append(poolid) pools.append(poolid)
pools = sorted(pools) pools = sorted(pools)
return pools return pools
def get_rrd_for_vm(proxmox, vmid, source, time):
node = proxmox.nodes(get_vm_node(proxmox, vmid))
image = node.qemu(vmid).rrd.get(ds=source, timeframe=time)['image']
return image

View file

@ -6,7 +6,8 @@ def connect_starrs():
try: try:
starrs = psycopg2.connect( starrs = psycopg2.connect(
"dbname='{}' user='{}' host='{}' password='{}'".format( "dbname='{}' user='{}' host='{}' password='{}'".format(
app.config['STARRS_DB_NAME'], app.config['STARRS_DB_USER'], app.config['STARRS_DB_HOST'], app.config['STARRS_DB_PASS'])) app.config['STARRS_DB_NAME'], app.config['STARRS_DB_USER'],
app.config['STARRS_DB_HOST'], app.config['STARRS_DB_PASS']))
except: except:
print("Unable to connect to STARRS database.") print("Unable to connect to STARRS database.")
raise raise

View file

@ -105,6 +105,37 @@ table, th, td {
} }
.usage-limit { .usage-limit {
width: 80%;
margin: 0px auto; margin: 0px auto;
} }
.rrd-graph {
width: 800px;
height: 200px;
padding-top: 10px;
padding-bottom: 10px;
}
.profile {
float: left;
width: 35%;
height: 60px;
}
.profile-img {
max-width: 100%;
max-height: 100%;
}
.progress {
margin-top: 2px;
margin-bottom: 0px;
float: right;
width: 85%;
height: 12px;
}
.resource-bar {
width: 60%;
float: right;
padding-bottom: 2px;
}

View file

@ -584,18 +584,21 @@ $(".edit-limit").click(function(){
cpu_text.innerHTML = 'CPU'; cpu_text.innerHTML = 'CPU';
options.append(cpu_text); options.append(cpu_text);
var cpu = document.createElement('input'); var cpu = document.createElement('input');
cpu.type = 'number';
cpu.defaultValue = cur_cpu; cpu.defaultValue = cur_cpu;
options.append(cpu); options.append(cpu);
mem_text = document.createElement('p'); mem_text = document.createElement('p');
mem_text.innerHTML = 'Memory (GB)'; mem_text.innerHTML = 'Memory (GB)';
options.append(mem_text); options.append(mem_text);
var mem = document.createElement('input'); var mem = document.createElement('input');
mem.type = 'number';
mem.defaultValue = cur_mem; mem.defaultValue = cur_mem;
options.append(mem) options.append(mem)
disk_text = document.createElement('p'); disk_text = document.createElement('p');
disk_text.innerHTML = 'Disk (GB)'; disk_text.innerHTML = 'Disk (GB)';
options.append(disk_text); options.append(disk_text);
var disk = document.createElement('input'); var disk = document.createElement('input');
disk.type = 'number';
disk.defaultValue = cur_disk; disk.defaultValue = cur_disk;
options.append(disk) options.append(disk)
swal({ swal({

View file

@ -27,6 +27,7 @@
<div class="form-group"> <div class="form-group">
<label for="mem">Memory</label> <label for="mem">Memory</label>
<select name="mem" id="mem" class="form-control"> <select name="mem" id="mem" class="form-control">
<option value="512">512MB</option>
{% for i in range(1, limits['mem'] + 1) %} {% for i in range(1, limits['mem'] + 1) %}
<option value="{{ i * 1024 }}">{{ i }}GB</option> <option value="{{ i * 1024 }}">{{ i }}GB</option>
{% endfor %} {% endfor %}

View file

@ -3,6 +3,15 @@
<div class="container"> <div class="container">
<div class="row"> <div class="row">
{% if rtp and rtp_view != True %}
<div class="col-md-12 col-sm-12">
<div class="panel panel-default">
<div class="panel-heading">
<p>{{ rtp_view }}</p>
</div>
</div>
</div>
{% endif %}
{% if not vms %} {% if not vms %}
<div class="col-md-12 col-sm-12"> <div class="col-md-12 col-sm-12">
<div class="panel panel-default"> <div class="panel panel-default">
@ -11,7 +20,7 @@
</div> </div>
</div> </div>
</div> </div>
{% elif not rtp %} {% elif rtp_view != True %}
{% for vm in vms %} {% for vm in vms %}
<div class="col-md-3 col-sm-4 col-xs-6"> <div class="col-md-3 col-sm-4 col-xs-6">
<div class="panel panel-default"> <div class="panel panel-default">
@ -26,29 +35,58 @@
{% endfor %} {% endfor %}
{% else %} {% else %}
{% for pool in vms %} {% for pool in vms %}
{% if pool[1] %} <div class="col-md-3 col-sm-4 col-xs-6">
<div class="col-md-12 col-sm-12">
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<h3 class="panel-title">{{ pool[0] }}</h3> <h3 class="panel-title">
<a href="/user/{{ pool['user'] }}">
<p>{{ pool['user'] }}</p>
</a>
</h3>
</div> </div>
<div class="panel-body"> <div class="panel-body">
{% for vm in pool[1] %} <div class="profile">
<div class="col-md-3 col-sm-4 col-xs-6"> <img class="profile-img" src="https://profiles.csh.rit.edu/image/{{ pool['user'] }}" title="{{ pool['user'] }}">
<div class="panel panel-default"> </div>
<div class="panel-body"> <div class="resource-bar">
<a href="/vm/{{ vm['vmid'] }}"> <span class="glyphicon glyphicon-cog pull-left"></span>
<p>{{ vm['name'] }}</p> <div class="progress">
</a> {% if pool['percents']['cpu'] <= 50 %}
<p>Status: {{ vm['status'] }}</p> <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="{{ pool['percents']['cpu'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['cpu'] }}%"></div>
</div> {% elif pool['percents']['cpu'] <= 75 %}
</div> <div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="{{ pool['percents']['cpu'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['cpu'] }}%"></div>
{% else %}
<div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="{{ pool['percents']['cpu'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['cpu'] }}%"></div>
{% endif %}
</div> </div>
{% endfor %} </div>
<div class="resource-bar">
<span class="glyphicon glyphicon-tasks pull-left"></span>
<div class="progress">
{% if pool['percents']['mem'] <= 50 %}
<div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="{{ pool['percents']['mem'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['mem'] }}%"></div>
{% elif pool['percents']['mem'] <= 75 %}
<div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="{{ pool['percents']['mem'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['mem'] }}%"></div>
{% else %}
<div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="{{ pool['percents']['mem'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['mem'] }}%"></div>
{% endif %}
</div>
</div>
<div class="resource-bar">
<span class="glyphicon glyphicon-hdd pull-left"></span>
<div class="progress">
{% if pool['percents']['disk'] <= 50 %}
<div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="{{ pool['percents']['disk'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['disk'] }}%"></div>
{% elif pool['percents']['disk'] <= 75 %}
<div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="{{ pool['percents']['disk'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['disk'] }}%"></div>
{% else %}
<div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="{{ pool['percents']['disk'] }}" aria-valuemin="0" aria-valuemax="100" style="width:{{ pool['percents']['disk'] }}%"></div>
{% endif %}
</div>
</div>
</div> </div>
</div> </div>
</div> </div>
{% endif %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
</div> </div>

View file

@ -96,7 +96,11 @@
</dd> </dd>
<dt>Memory</dt> <dt>Memory</dt>
<dd> <dd>
{% if vm['config']['memory'] < 1024 %}
{{ vm['config']['memory'] }}MB
{% else %}
{{ vm['config']['memory'] // 1024 }}GB {{ vm['config']['memory'] // 1024 }}GB
{% endif %}
{% if vm['qmpstatus'] == 'running' or vm['qmpstatus'] == 'paused' %} {% if vm['qmpstatus'] == 'running' or vm['qmpstatus'] == 'paused' %}
<button class="btn btn-default proxstar-changebtn" id="change-mem" data-vmid="{{ vm['vmid'] }}" data-mem="{{ vm['config']['memory'] // 1024 }}" data-usage="{{ usage['mem'] - vm['config']['memory'] // 1024 }}" data-limit="{{ limits['mem'] }}"> <button class="btn btn-default proxstar-changebtn" id="change-mem" data-vmid="{{ vm['vmid'] }}" data-mem="{{ vm['config']['memory'] // 1024 }}" data-usage="{{ usage['mem'] - vm['config']['memory'] // 1024 }}" data-limit="{{ limits['mem'] }}">
<span class="glyphicon glyphicon-cog"></span> <span class="glyphicon glyphicon-cog"></span>
@ -118,6 +122,21 @@
</div> </div>
</div> </div>
</div> </div>
<div class="col-md-12 col-sm-12">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">RRD Graphs</h3>
</div>
<div class="panel-body">
<img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/cpu.png">
<img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/mem.png">
<img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/netin.png">
<img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/netout.png">
<img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/diskread.png">
<img class="rrd-graph" src="/vm/{{ vm['vmid'] }}/rrd/diskwrite.png">
</div>
</div>
</div>
</div> </div>
</div> </div>