diff --git a/gunicorn_conf.py b/gunicorn_conf.py index 70a8ce0..3dee104 100644 --- a/gunicorn_conf.py +++ b/gunicorn_conf.py @@ -1,5 +1,6 @@ import os import subprocess + from flask import Flask app = Flask(__name__) diff --git a/proxstar/__init__.py b/proxstar/__init__.py index c877ca2..d1067a3 100644 --- a/proxstar/__init__.py +++ b/proxstar/__init__.py @@ -3,6 +3,7 @@ import json import time import psutil import atexit +import logging import psycopg2 import subprocess import rq_dashboard @@ -20,6 +21,9 @@ from proxstar.starrs import * from proxstar.ldapdb import * from proxstar.proxmox import * +logging.basicConfig( + format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) + app = Flask(__name__) app.config.from_object(rq_dashboard.default_settings) if os.path.exists( @@ -38,10 +42,16 @@ with open('proxmox_ssh_key', 'w') as key: ssh_tunnels = [] -auth = OIDCAuthentication( - app, - issuer=app.config['OIDC_ISSUER'], - client_registration_info=app.config['OIDC_CLIENT_CONFIG']) +# Keep on retrying until we have auth defined since SSO sucks +while True: + try: + auth + break + except: + auth = OIDCAuthentication( + app, + issuer=app.config['OIDC_ISSUER'], + client_registration_info=app.config['OIDC_CLIENT_CONFIG']) redis_conn = Redis(app.config['REDIS_HOST'], app.config['REDIS_PORT']) q = Queue(connection=redis_conn) @@ -62,6 +72,7 @@ from proxstar.user import User from proxstar.tasks import generate_pool_cache_task, process_expiring_vms_task, cleanup_vnc_task, delete_vm_task, create_vm_task, setup_template_task if 'generate_pool_cache' not in scheduler: + logging.info('adding generate pool cache task to scheduler') scheduler.schedule( id='generate_pool_cache', scheduled_time=datetime.datetime.utcnow(), @@ -69,10 +80,12 @@ if 'generate_pool_cache' not in scheduler: interval=90) if 'process_expiring_vms' not in scheduler: + logging.info('adding process expiring VMs task to scheduler') scheduler.cron( '0 5 * * *', id='process_expiring_vms', func=process_expiring_vms_task) if 'cleanup_vnc' not in scheduler: + logging.info('adding cleanup VNC task to scheduler') scheduler.schedule( id='cleanup_vnc', scheduled_time=datetime.datetime.utcnow(), @@ -238,7 +251,8 @@ def vm_console(vmid): port = str(5900 + int(vmid)) token = add_vnc_target(port) node = "{}.csh.rit.edu".format(vm.node) - print("Creating SSH tunnel to {} for VM {}.".format(node, vm.id)) + logging.info("Creating SSH tunnel to {} for VM {}.".format( + node, vm.id)) tunnel = start_ssh_tunnel(node, port) ssh_tunnels.append(tunnel) vm.start_vnc(port) diff --git a/proxstar/db.py b/proxstar/db.py index f09fef5..3c5a89d 100644 --- a/proxstar/db.py +++ b/proxstar/db.py @@ -1,8 +1,11 @@ import datetime -from sqlalchemy import exists + from dateutil.relativedelta import relativedelta +from sqlalchemy import exists + from proxstar.ldapdb import * -from proxstar.models import VM_Expiration, Usage_Limit, Pool_Cache, Ignored_Pools, Template, Allowed_Users, Base +from proxstar.models import (Allowed_Users, Base, Ignored_Pools, Pool_Cache, + Template, Usage_Limit, VM_Expiration) def get_vm_expire(db, vmid, months): diff --git a/proxstar/ldapdb.py b/proxstar/ldapdb.py index d9dac24..a7c5fcd 100644 --- a/proxstar/ldapdb.py +++ b/proxstar/ldapdb.py @@ -1,12 +1,14 @@ from csh_ldap import CSHLDAP from flask import current_app as app +from proxstar import logging + def connect_ldap(): try: ldap = CSHLDAP(app.config['LDAP_BIND_DN'], app.config['LDAP_BIND_PW']) - except: - print("Unable to connect to LDAP.") + except Exception as e: + logging.error("unable to connect to LDAP: %s", e) raise return ldap diff --git a/proxstar/mail.py b/proxstar/mail.py index 9b21895..10721b6 100644 --- a/proxstar/mail.py +++ b/proxstar/mail.py @@ -1,7 +1,7 @@ import smtplib -from email.utils import formatdate -from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formatdate def send_email(toaddr, subject, body): diff --git a/proxstar/models.py b/proxstar/models.py index 45bf2ba..970ec92 100644 --- a/proxstar/models.py +++ b/proxstar/models.py @@ -1,7 +1,7 @@ -from sqlalchemy import Column, Integer, String, Date -from sqlalchemy.types import JSON, Text +from sqlalchemy import Column, Date, Integer, String from sqlalchemy.dialects import postgresql from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.types import JSON, Text Base = declarative_base() diff --git a/proxstar/proxmox.py b/proxstar/proxmox.py index b7d9ede..3d1ebe9 100644 --- a/proxstar/proxmox.py +++ b/proxstar/proxmox.py @@ -1,7 +1,9 @@ -from proxmoxer import ProxmoxAPI -from proxstar.ldapdb import is_user -from proxstar.db import get_user_usage_limits, get_ignored_pools from flask import current_app as app +from proxmoxer import ProxmoxAPI + +from proxstar import logging +from proxstar.db import get_ignored_pools, get_user_usage_limits +from proxstar.ldapdb import is_user def connect_proxmox(): @@ -17,7 +19,8 @@ def connect_proxmox(): except: if app.config['PROXMOX_HOSTS'].index(host) == ( len(app.config['PROXMOX_HOSTS']) - 1): - print('Unable to connect to any of the given Proxmox servers!') + logging.error( + 'unable to connect to any of the given Proxmox servers') raise @@ -35,7 +38,8 @@ def connect_proxmox_ssh(): except: if app.config['PROXMOX_HOSTS'].index(host) == ( len(app.config['PROXMOX_HOSTS']) - 1): - print('Unable to connect to any of the given Proxmox servers!') + logging.error( + 'unable to connect to any of the given Proxmox servers') raise diff --git a/proxstar/tasks.py b/proxstar/tasks.py index 2886111..fd3edda 100644 --- a/proxstar/tasks.py +++ b/proxstar/tasks.py @@ -1,20 +1,26 @@ +import logging import os import time -import requests + import paramiko import psycopg2 +import requests from flask import Flask from rq import get_current_job from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker + from proxstar.db import * -from proxstar.util import * from proxstar.mail import * -from proxstar.starrs import * -from proxstar.vnc import send_stop_ssh_tunnel -from proxstar.vm import VM, create_vm, clone_vm -from proxstar.user import User, get_vms_for_rtp from proxstar.proxmox import connect_proxmox, get_pools +from proxstar.starrs import * +from proxstar.user import User, get_vms_for_rtp +from proxstar.util import * +from proxstar.vm import VM, clone_vm, create_vm +from proxstar.vnc import send_stop_ssh_tunnel + +logging.basicConfig( + format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) app = Flask(__name__) if os.path.exists( @@ -54,10 +60,11 @@ def create_vm_task(user, name, cores, memory, disk, iso): proxmox = connect_proxmox() db = connect_db() starrs = connect_starrs() - print("[{}] Creating VM.".format(name)) + logging.info("[{}] Creating VM.".format(name)) set_job_status(job, 'creating VM') vmid = create_vm(proxmox, user, name, cores, memory, disk, iso) - print("[{}] Waiting until Proxmox is done provisioning.".format(name)) + logging.info( + "[{}] Waiting until Proxmox is done provisioning.".format(name)) set_job_status(job, 'waiting for Proxmox') timeout = 20 retry = 0 @@ -68,11 +75,11 @@ def create_vm_task(user, name, cores, memory, disk, iso): continue break if retry == timeout: - print("[{}] Failed to provision, deleting.".format(name)) + logging.info("[{}] Failed to provision, deleting.".format(name)) set_job_status(job, 'failed to provision') delete_vm_task(vmid) return - print("[{}] Registering in STARRS.".format(name)) + logging.info("[{}] Registering in STARRS.".format(name)) set_job_status(job, 'registering in STARRS') vm = VM(vmid) ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE']) @@ -80,7 +87,7 @@ def create_vm_task(user, name, cores, memory, disk, iso): ip) set_job_status(job, 'setting VM expiration') get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS']) - print("[{}] VM successfully provisioned.".format(name)) + logging.info("[{}] VM successfully provisioned.".format(name)) set_job_status(job, 'complete') @@ -123,7 +130,7 @@ def process_expiring_vms_task(): expired_vms.append([vm.id, vm.name, days]) vm.stop() elif days <= -7: - print( + logging.info( "Deleting {} ({}) as it has been at least a week since expiration." .format(vm.name, vm.id)) send_stop_ssh_tunnel(vm.id) @@ -148,13 +155,14 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory): proxmox = connect_proxmox() starrs = connect_starrs() db = connect_db() - print("[{}] Retrieving template info for template {}.".format( + logging.info("[{}] Retrieving template info for template {}.".format( name, template_id)) template = get_template(db, template_id) - print("[{}] Cloning template {}.".format(name, template_id)) + logging.info("[{}] Cloning template {}.".format(name, template_id)) set_job_status(job, 'cloning template') vmid = clone_vm(proxmox, template_id, name, user) - print("[{}] Waiting until Proxmox is done provisioning.".format(name)) + logging.info( + "[{}] Waiting until Proxmox is done provisioning.".format(name)) set_job_status(job, 'waiting for Proxmox') timeout = 20 retry = 0 @@ -165,36 +173,37 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory): continue break if retry == timeout: - print("[{}] Failed to provision, deleting.".format(name)) + logging.info("[{}] Failed to provision, deleting.".format(name)) set_job_status(job, 'failed to provision') delete_vm_task(vmid) return - print("[{}] Registering in STARRS.".format(name)) + logging.info("[{}] Registering in STARRS.".format(name)) set_job_status(job, 'registering in STARRS') vm = VM(vmid) ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE']) register_starrs(starrs, name, app.config['STARRS_USER'], vm.get_mac(), ip) get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS']) - print("[{}] Setting CPU and memory.".format(name)) + logging.info("[{}] Setting CPU and memory.".format(name)) set_job_status(job, 'setting CPU and memory') vm.set_cpu(cores) vm.set_mem(memory) - print("[{}] Applying cloud-init config.".format(name)) + logging.info("[{}] Applying cloud-init config.".format(name)) set_job_status(job, 'applying cloud-init') vm.set_ci_user(user) vm.set_ci_ssh_key(ssh_key) vm.set_ci_network() - print("[{}] Waiting for STARRS to propogate before starting VM.". - format(name)) + logging.info( + "[{}] Waiting for STARRS to propogate before starting VM.".format( + name)) set_job_status(job, 'waiting for STARRS') job.save_meta() time.sleep(90) - print("[{}] Starting VM.".format(name)) + logging.info("[{}] Starting VM.".format(name)) set_job_status(job, 'starting VM') job.save_meta() vm.start() - print("[{}] Template successfully provisioned.".format(name)) + logging.info("[{}] Template successfully provisioned.".format(name)) set_job_status(job, 'completed') job.save_meta() diff --git a/proxstar/user.py b/proxstar/user.py index 6bda641..a707dd8 100644 --- a/proxstar/user.py +++ b/proxstar/user.py @@ -1,10 +1,11 @@ +from proxmoxer.core import ResourceException +from rq.registry import StartedJobRegistry + from proxstar import db, q, redis_conn from proxstar.db import * -from proxstar.vm import VM -from proxstar.util import * from proxstar.proxmox import * -from rq.registry import StartedJobRegistry -from proxmoxer.core import ResourceException +from proxstar.util import * +from proxstar.vm import VM class User(object): @@ -23,7 +24,7 @@ class User(object): vms = proxmox.pools(self.name).get()['members'] except ResourceException: # they likely don't have a pool yet, try to create it - if is_user(self.name) and not is_rtp(self.name): + if is_user(self.name): proxmox.pools.post( poolid=self.name, comment='Managed by Proxstar') # if created, their pool is empty so return empty array diff --git a/proxstar/util.py b/proxstar/util.py index b4eedf5..598614a 100644 --- a/proxstar/util.py +++ b/proxstar/util.py @@ -1,5 +1,5 @@ -import string import random +import string def gen_password( diff --git a/proxstar/vm.py b/proxstar/vm.py index 343231d..c60a321 100644 --- a/proxstar/vm.py +++ b/proxstar/vm.py @@ -1,13 +1,16 @@ -import time import json +import time import urllib -from tenacity import retry, wait_fixed, stop_after_attempt -from proxstar import db, starrs -from proxstar.db import get_vm_expire, delete_vm_expire -from proxstar.util import lazy_property -from proxstar.starrs import get_ip_for_mac -from proxstar.proxmox import connect_proxmox, connect_proxmox_ssh, get_node_least_mem, get_free_vmid, get_vm_node + from flask import current_app as app +from tenacity import retry, stop_after_attempt, wait_fixed + +from proxstar import db, starrs +from proxstar.db import delete_vm_expire, get_vm_expire +from proxstar.proxmox import (connect_proxmox, connect_proxmox_ssh, + get_free_vmid, get_node_least_mem, get_vm_node) +from proxstar.starrs import get_ip_for_mac +from proxstar.util import lazy_property class VM(object): diff --git a/proxstar/vnc.py b/proxstar/vnc.py index fd5439e..d2cf149 100644 --- a/proxstar/vnc.py +++ b/proxstar/vnc.py @@ -1,10 +1,13 @@ import os -import time -import requests import subprocess -from sshtunnel import SSHTunnelForwarder -from proxstar.util import * +import time + +import requests from flask import current_app as app +from sshtunnel import SSHTunnelForwarder + +from proxstar import logging +from proxstar.util import * def stop_websockify(): @@ -18,7 +21,7 @@ def stop_websockify(): time.sleep(10) if subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE).stdout: - print('Websockify didn\'t stop, killing forcefully.') + logging.info('websockify didn\'t stop, killing forcefully.') subprocess.run(['kill', '-9', pid], stdout=subprocess.PIPE) @@ -83,7 +86,7 @@ def stop_ssh_tunnel(vmid, ssh_tunnels): (tunnel for tunnel in ssh_tunnels if tunnel.local_bind_port == port), None) if tunnel: - print("Tearing down SSH tunnel for VM {}.".format(vmid)) + logging.info("Tearing down SSH tunnel for VM {}.".format(vmid)) try: tunnel.stop() except: