diff --git a/gunicorn_conf.py b/gunicorn_conf.py index a389243..70a8ce0 100644 --- a/gunicorn_conf.py +++ b/gunicorn_conf.py @@ -16,12 +16,11 @@ app.config.from_pyfile(config) def start_websockify(websockify_path, target_file): result = subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE) if not result.stdout: - subprocess.call( - [ - websockify_path, '8081', '--token-plugin', 'TokenFile', - '--token-source', target_file, '-D' - ], - stdout=subprocess.PIPE) + subprocess.call([ + websockify_path, '8081', '--token-plugin', 'TokenFile', + '--token-source', target_file, '-D' + ], + stdout=subprocess.PIPE) def on_starting(server): diff --git a/proxstar/__init__.py b/proxstar/__init__.py index 26e8bc8..e9254f5 100644 --- a/proxstar/__init__.py +++ b/proxstar/__init__.py @@ -140,8 +140,9 @@ def list_vms(user_view=None): if user.active: vms = user.vms for pending_vm in user.pending_vms: - vm = next((vm for vm in vms - if vm['name'] == pending_vm['name']), None) + vm = next( + (vm for vm in vms if vm['name'] == pending_vm['name']), + None) if vm: vms[vms.index(vm)]['status'] = pending_vm['status'] vms[vms.index(vm)]['pending'] = True diff --git a/proxstar/tasks.py b/proxstar/tasks.py index 07b471a..fd469dc 100644 --- a/proxstar/tasks.py +++ b/proxstar/tasks.py @@ -104,8 +104,8 @@ def process_expiring_vms_task(): vm.stop() elif days <= -7: print( - "Deleting {} ({}) as it has been at least a week since expiration.". - format(vm.name, vm.id)) + "Deleting {} ({}) as it has been at least a week since expiration." + .format(vm.name, vm.id)) send_stop_ssh_tunnel(vm.id) delete_vm_task(vm.id) if expiring_vms: @@ -135,15 +135,29 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory): job.meta['status'] = 'cloning template' job.save_meta() vmid, mac = clone_vm(proxmox, template_id, name, user) + print("[{}] Waiting until Proxmox is done provisioning.".format(name)) + job.meta['status'] = 'waiting for Proxmox' + job.save_meta() + timeout = 200 + retry = 0 + while retry < timeout: + if not VM(vmid).is_provisioned(): + retry += 1 + time.sleep(3) + continue + break + if retry == timeout: + print("[{}] Failed to provision, deleting.".format(name)) + job.meta['status'] = 'failed to provision' + job.save_meta() + delete_vm_task(vmid) + return print("[{}] Registering in STARRS.".format(name)) job.meta['status'] = 'registering in STARRS' job.save_meta() ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE']) register_starrs(starrs, name, app.config['STARRS_USER'], mac, ip) get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS']) - print("[{}] Giving Proxmox some time to finish cloning.".format(name)) - job.meta['status'] = 'waiting for Proxmox' - time.sleep(15) print("[{}] Setting CPU and memory.".format(name)) job.meta['status'] = 'setting CPU and memory' job.save_meta() @@ -152,12 +166,12 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory): vm.set_mem(memory) print("[{}] Applying cloud-init config.".format(name)) job.meta['status'] = 'applying cloud-init' + job.save_meta() vm.set_ci_user(user) vm.set_ci_ssh_key(ssh_key) vm.set_ci_network() - print( - "[{}] Waiting for STARRS to propogate before starting VM.".format( - name)) + print("[{}] Waiting for STARRS to propogate before starting VM.". + format(name)) job.meta['status'] = 'waiting for STARRS' job.save_meta() time.sleep(90) diff --git a/proxstar/vm.py b/proxstar/vm.py index d8f877f..4592654 100644 --- a/proxstar/vm.py +++ b/proxstar/vm.py @@ -37,6 +37,13 @@ class VM(object): def qmpstatus(self): return self.info['qmpstatus'] + def is_provisioned(self): + try: + self.set_cpu(self.cpu) + return True + except: + return False + @lazy_property def node(self): proxmox = connect_proxmox() diff --git a/proxstar/vnc.py b/proxstar/vnc.py index b16358f..8d290a4 100644 --- a/proxstar/vnc.py +++ b/proxstar/vnc.py @@ -10,12 +10,11 @@ from flask import current_app as app def start_websockify(websockify_path, target_file): result = subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE) if not result.stdout: - subprocess.call( - [ - websockify_path, '8081', '--token-plugin', 'TokenFile', - '--token-source', target_file, '-D' - ], - stdout=subprocess.PIPE) + subprocess.call([ + websockify_path, '8081', '--token-plugin', 'TokenFile', + '--token-source', target_file, '-D' + ], + stdout=subprocess.PIPE) def stop_websockify(): @@ -24,11 +23,11 @@ def stop_websockify(): pid = result.stdout.strip() subprocess.run(['kill', pid], stdout=subprocess.PIPE) time.sleep(3) - if subprocess.run( - ['pgrep', 'websockify'], stdout=subprocess.PIPE).stdout: + if subprocess.run(['pgrep', 'websockify'], + stdout=subprocess.PIPE).stdout: time.sleep(10) - if subprocess.run( - ['pgrep', 'websockify'], stdout=subprocess.PIPE).stdout: + if subprocess.run(['pgrep', 'websockify'], + stdout=subprocess.PIPE).stdout: print('Websockify didn\'t stop, killing forcefully.') subprocess.run(['kill', '-9', pid], stdout=subprocess.PIPE) @@ -90,8 +89,9 @@ def start_ssh_tunnel(node, port): def stop_ssh_tunnel(vmid, ssh_tunnels): # Tear down the SSH tunnel and VNC target entry for a given VM port = 5900 + int(vmid) - tunnel = next((tunnel for tunnel in ssh_tunnels - if tunnel.local_bind_port == port), None) + tunnel = next( + (tunnel for tunnel in ssh_tunnels if tunnel.local_bind_port == port), + None) if tunnel: print("Tearing down SSH tunnel for VM {}.".format(vmid)) try: