fix expire bug, cleanup tasks

proxstar/tasks.py

@@ -43,25 +43,45 @@ def connect_starrs():
     return starrs
 
 
+def set_job_status(job, status):
+    job.meta['status'] = status
+    job.save_meta()
+
+
 def create_vm_task(user, name, cores, memory, disk, iso):
     with app.app_context():
         job = get_current_job()
         proxmox = connect_proxmox()
         db = connect_db()
         starrs = connect_starrs()
-        job.meta['status'] = 'creating VM'
+        print("[{}] Creating VM.".format(name))
-        job.save_meta()
+        set_job_status(job, 'creating VM')
-        vmid, mac = create_vm(proxmox, user, name, cores, memory, disk, iso)
+        vmid = create_vm(proxmox, user, name, cores, memory, disk, iso)
-        job.meta['status'] = 'registering in STARRS'
+        print("[{}] Waiting until Proxmox is done provisioning.".format(name))
-        job.save_meta()
+        set_job_status(job, 'waiting for Proxmox')
-        register_starrs(starrs, name, app.config['STARRS_USER'], mac,
+        timeout = 20
-                        get_next_ip(starrs, app.config['STARRS_IP_RANGE']))
+        retry = 0
-        job.meta['status'] = 'setting VM expiration'
+        while retry < timeout:
-        job.save_meta()
+            if not VM(vmid).is_provisioned():
-        delete_vm_expire(db, vmid)
+                retry += 1
+                time.sleep(3)
+                continue
+            break
+        if retry == timeout:
+            print("[{}] Failed to provision, deleting.".format(name))
+            set_job_status(job, 'failed to provision')
+            delete_vm_task(vmid)
+            return
+        print("[{}] Registering in STARRS.".format(name))
+        set_job_status(job, 'registering in STARRS')
+        vm = VM(vmid)
+        ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
+        register_starrs(starrs, name, app.config['STARRS_USER'], vm.get_mac(),
+                        ip)
+        set_job_status(job, 'setting VM expiration')
         get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
-        job.meta['status'] = 'complete'
+        print("[{}] VM successfully provisioned.".format(name))
-        job.save_meta()
+        set_job_status(job, 'complete')
 
 
 def delete_vm_task(vmid):
@@ -78,8 +98,8 @@ def delete_vm_task(vmid):
                     break
                 retry += 1
         vm.delete()
-        delete_starrs(starrs, vm.name)
         delete_vm_expire(db, vmid)
+        delete_starrs(starrs, vm.name)
 
 
 def process_expiring_vms_task():
@@ -132,12 +152,10 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory):
             name, template_id))
         template = get_template(db, template_id)
         print("[{}] Cloning template {}.".format(name, template_id))
-        job.meta['status'] = 'cloning template'
+        set_job_status(job, 'cloning template')
-        job.save_meta()
+        vmid = clone_vm(proxmox, template_id, name, user)
-        vmid, mac = clone_vm(proxmox, template_id, name, user)
         print("[{}] Waiting until Proxmox is done provisioning.".format(name))
-        job.meta['status'] = 'waiting for Proxmox'
+        set_job_status(job, 'waiting for Proxmox')
-        job.save_meta()
         timeout = 20
         retry = 0
         while retry < timeout:
@@ -148,39 +166,36 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory):
             break
         if retry == timeout:
             print("[{}] Failed to provision, deleting.".format(name))
-            job.meta['status'] = 'failed to provision'
+            set_job_status(job, 'failed to provision')
-            job.save_meta()
             delete_vm_task(vmid)
             return
         print("[{}] Registering in STARRS.".format(name))
-        job.meta['status'] = 'registering in STARRS'
+        set_job_status(job, 'registering in STARRS')
-        job.save_meta()
+        vm = VM(vmid)
         ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
-        register_starrs(starrs, name, app.config['STARRS_USER'], mac, ip)
+        register_starrs(starrs, name, app.config['STARRS_USER'], vm.get_mac(),
+                        ip)
         get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
         print("[{}] Setting CPU and memory.".format(name))
-        job.meta['status'] = 'setting CPU and memory'
+        set_job_status(job, 'setting CPU and memory')
-        job.save_meta()
-        vm = VM(vmid)
         vm.set_cpu(cores)
         vm.set_mem(memory)
         print("[{}] Applying cloud-init config.".format(name))
-        job.meta['status'] = 'applying cloud-init'
+        set_job_status(job, 'applying cloud-init')
-        job.save_meta()
         vm.set_ci_user(user)
         vm.set_ci_ssh_key(ssh_key)
         vm.set_ci_network()
         print("[{}] Waiting for STARRS to propogate before starting VM.".
               format(name))
-        job.meta['status'] = 'waiting for STARRS'
+        set_job_status(job, 'waiting for STARRS')
         job.save_meta()
         time.sleep(90)
         print("[{}] Starting VM.".format(name))
-        job.meta['status'] = 'starting VM'
+        set_job_status(job, 'starting VM')
         job.save_meta()
         vm.start()
         print("[{}] Template successfully provisioned.".format(name))
-        job.meta['status'] = 'completed'
+        set_job_status(job, 'completed')
         job.save_meta()
 
 

proxstar/vm.py

@@ -3,7 +3,7 @@ import json
 import urllib
 from tenacity import retry, wait_fixed, stop_after_attempt
 from proxstar import db, starrs
-from proxstar.db import get_vm_expire
+from proxstar.db import get_vm_expire, delete_vm_expire
 from proxstar.util import lazy_property
 from proxstar.starrs import get_ip_for_mac
 from proxstar.proxmox import connect_proxmox, connect_proxmox_ssh, get_node_least_mem, get_free_vmid, get_vm_node
@@ -248,9 +248,13 @@ class VM(object):
         proxmox.nodes(self.node).qemu(self.id).config.put(ipconfig0='ip=dhcp')
 
 
+# Will create a new VM with the given parameters, does not guarantee
+# the VM is done provisioning when returning
 def create_vm(proxmox, user, name, cores, memory, disk, iso):
     node = proxmox.nodes(get_node_least_mem(proxmox))
     vmid = get_free_vmid(proxmox)
+    # Make sure lingering expirations are deleted
+    delete_vm_expire(db, vmid)
     node.qemu.create(
         vmid=vmid,
         name=name,
@@ -262,34 +266,22 @@ def create_vm(proxmox, user, name, cores, memory, disk, iso):
         net0='virtio,bridge=vmbr0',
         pool=user,
         description='Managed by Proxstar')
-    retry = 0
+    return vmid
-    while retry < 20:
-        try:
-            mac = VM(vmid).get_mac()
-            break
-        except:
-            retry += 1
-            time.sleep(3)
-    return vmid, mac
 
 
+# Will clone a new VM from a template, does not guarantee the
+# VM is done provisioning when returning
 def clone_vm(proxmox, template_id, name, pool):
     node = proxmox.nodes(get_vm_node(proxmox, template_id))
-    newid = get_free_vmid(proxmox)
+    vmid = get_free_vmid(proxmox)
+    # Make sure lingering expirations are deleted
+    delete_vm_expire(db, vmid)
     target = get_node_least_mem(proxmox)
     node.qemu(template_id).clone.post(
-        newid=newid,
+        newid=vmid,
         name=name,
         pool=pool,
         full=1,
         description='Managed by Proxstar',
         target=target)
-    retry = 0
+    return vmid
-    while retry < 100:
-        try:
-            mac = VM(newid).get_mac()
-            break
-        except:
-            retry += 1
-            time.sleep(3)
-    return newid, mac

proxstar/vnc.py

@@ -7,16 +7,6 @@ from proxstar.util import *
 from flask import current_app as app
 
 
-def start_websockify(websockify_path, target_file):
-    result = subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE)
-    if not result.stdout:
-        subprocess.call([
-            websockify_path, '8081', '--token-plugin', 'TokenFile',
-            '--token-source', target_file, '-D'
-        ],
-                        stdout=subprocess.PIPE)
-
-
 def stop_websockify():
     result = subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE)
     if result.stdout: