add scheduler script, store rtp view cache in db and schedule generation

proxstar/__init__.py

@@ -46,6 +46,17 @@ starrs = psycopg2.connect(
         app.config['STARRS_DB_NAME'], app.config['STARRS_DB_USER'],
         app.config['STARRS_DB_HOST'], app.config['STARRS_DB_PASS']))
 
+if 'generate_pool_cache' not in scheduler:
+    scheduler.schedule(
+        id='generate_pool_cache',
+        scheduled_time=datetime.datetime.utcnow(),
+        func=generate_pool_cache_task,
+        interval=60)
+
+if 'process_expiring_vms' not in scheduler:
+    scheduler.cron(
+        '0 0 * * *', id='process_expiring_vms', func=process_expiring_vms_task)
+
 
 @app.route("/")
 @app.route("/user/<string:user>")
@@ -63,10 +74,7 @@ def list_vms(user=None):
         user = session['userinfo']['preferred_username']
     elif rtp:
         user = session['userinfo']['preferred_username']
-        vms = cache.get('vms')
+        vms = get_pool_cache(db)
-        if vms is None:
-            vms = get_vms_for_rtp(proxmox, db)
-            cache.set('vms', vms, timeout=5 * 60)
         rtp_view = True
     else:
         user = session['userinfo']['preferred_username']

proxstar/db.py

@@ -2,7 +2,7 @@ import datetime
 from sqlalchemy import exists
 from dateutil.relativedelta import relativedelta
 from proxstar.ldapdb import *
-from proxstar.models import VM_Expiration, Usage_Limit, Base
+from proxstar.models import VM_Expiration, Usage_Limit, Pool_Cache, Base
 
 
 def get_vm_expire(db, vmid, months):
@@ -95,3 +95,32 @@ def delete_user_usage_limits(db, user):
         limits = db.query(Usage_Limit).filter(Usage_Limit.id == user).one()
         db.delete(limits)
         db.commit()
+
+
+def store_pool_cache(db, pools):
+    db.query(Pool_Cache).delete()
+    for pool in pools:
+        pool_entry = Pool_Cache(
+            pool=pool['user'],
+            vms=pool['vms'],
+            num_vms=pool['num_vms'],
+            usage=pool['usage'],
+            limits=pool['limits'],
+            percents=pool['percents'])
+        db.add(pool_entry)
+    db.commit()
+
+
+def get_pool_cache(db):
+    db_pools = db.query(Pool_Cache).all()
+    pools = []
+    for pool in db_pools:
+        pool_dict = dict()
+        pool_dict['user'] = pool.pool
+        pool_dict['vms'] = pool.vms
+        pool_dict['num_vms'] = pool.num_vms
+        pool_dict['usage'] = pool.usage
+        pool_dict['limits'] = pool.limits
+        pool_dict['percents'] = pool.percents
+        pools.append(pool_dict)
+    return pools

proxstar/models.py

@@ -1,4 +1,6 @@
 from sqlalchemy import Column, Integer, String, Date
+from sqlalchemy.types import JSON, Text
+from sqlalchemy.dialects import postgresql
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
@@ -16,3 +18,13 @@ class Usage_Limit(Base):
     cpu = Column(Integer, nullable=False)
     mem = Column(Integer, nullable=False)
     disk = Column(Integer, nullable=False)
+
+
+class Pool_Cache(Base):
+    __tablename__ = 'pool_cache'
+    pool = Column(String(32), primary_key=True)
+    vms = Column(postgresql.ARRAY(Text, dimensions=2), nullable=False)
+    num_vms = Column(Integer, nullable=False)
+    usage = Column(JSON, nullable=False)
+    limits = Column(JSON, nullable=False)
+    percents = Column(JSON, nullable=False)

proxstar/proxmox.py

@@ -40,9 +40,8 @@ def get_vms_for_user(proxmox, user):
 
 
 def get_vms_for_rtp(proxmox, db):
-    pools = get_pools(proxmox)
+    pools = []
-    pool_vms = []
+    for pool in get_pools(proxmox):
-    for pool in pools:
         pool_dict = dict()
         pool_dict['user'] = pool
         pool_dict['vms'] = get_vms_for_user(proxmox, pool)
@@ -51,8 +50,8 @@ def get_vms_for_rtp(proxmox, db):
         pool_dict['limits'] = get_user_usage_limits(db, pool)
         pool_dict['percents'] = get_user_usage_percent(
             proxmox, pool, pool_dict['usage'], pool_dict['limits'])
-        pool_vms.append(pool_dict)
+        pools.append(pool_dict)
-    return pool_vms
+    return pools
 
 
 def get_user_allowed_vms(proxmox, user):

proxstar/tasks.py

@@ -90,3 +90,11 @@ def process_expiring_vms_task():
                     expiring_vms.append([name, days])
             if expiring_vms:
                 send_vm_expire_email('com6056', expiring_vms)
+
+
+def generate_pool_cache_task():
+    with app.app_context():
+        proxmox = connect_proxmox()
+        db = connect_db()
+        pools = get_vms_for_rtp(proxmox, db)
+        store_pool_cache(db, pools)

start_scheduler.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+/opt/app-root/bin/rq worker -u "$PROXSTAR_REDIS_URL"