Merge branch 'master' into increase-default-limits

.gitattributes

@@ -0,0 +1 @@
+HACKING/proxstar-postgres/schema/large.sql linguist-generated

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8-buster
+WORKDIR /opt/proxstar
+RUN apt-get update -y && apt-get install -y python3-dev libldap2-dev libsasl2-dev ldap-utils
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY start_worker.sh start_scheduler.sh .
+COPY .git ./.git
+COPY *.py .
+COPY proxstar ./proxstar
+RUN touch proxmox_ssh_key && chmod a+w proxmox_ssh_key # This is some OKD shit.
+ENTRYPOINT ddtrace-run python3 wsgi.py

HACKING/.env.template

@@ -0,0 +1,67 @@
+# Proxstar
+PROXSTAR_VM_EXPIRE_MONTHS=3
+PROXSTAR_VNC_CLEANUP_TOKEN=
+
+# Development options
+# Determines weather or not to run STARRS queries (for doing stuff like checking for available IPs)
+PROXSTAR_USE_STARRS=True
+# If you're an RTP and want to see a normal user's homepage view, set this to True.
+PROXSTAR_FORCE_STANDARD_USER=False
+
+# Flask
+PROXSTAR_IP=0.0.0.0 # The IP address to which Proxstar is served.
+PROXSTAR_PORT=8000 # The port Proxstar runs on.
+PROXSTAR_SERVER_NAME=localhost:8000 # It's configured this way in SSO.
+PROXSTAR_SECRET_KEY= # This can be LITERALLY anything.
+
+# OIDC
+PROXSTAR_OIDC_ISSUER=https://sso.csh.rit.edu/auth/realms/csh
+PROXSTAR_CLIENT_ID= # Ask an RTP. Can get this from OKD configs.
+PROXSTAR_CLIENT_SECRET= # Ask an RTP.
+PROXSTAR_REDIRECT_URI=https://proxstar.csh.rit.edu/logout
+
+# Proxmox
+PROXSTAR_PROXMOX_HOSTS= # Host list that Proxstar can use (proxstar01, etc...)
+PROXSTAR_PROXMOX_USER=api@pve
+PROXSTAR_PROXMOX_PASS= # Password for proxstar user
+PROXSTAR_PROXMOX_ISO_STORAGE=nfs-iso
+PROXSTAR_PROXMOX_VM_STORAGE=ceph
+PROXSTAR_PROXMOX_SSH_USER=root
+PROXSTAR_PROXMOX_SSH_KEY="" # Ask an RTP. This is gonna look like a certificate.
+PROXSTAR_PROXMOX_SSH_KEY_PASS= # Password for above certificate
+
+# STARRS
+PROXSTAR_STARRS_DB_HOST=proxstar-postgres
+PROXSTAR_DB_NAME=starrs
+PROXSTAR_DB_USER=postgres
+PROXSTAR_DB_PASS=changeme
+PROXSTAR_STARRS_USER=proxstar
+PROXSTAR_IP_RANGE=
+
+# LDAP
+PROXSTAR_LDAP_BIND_DN= # Ask an RTP. Can be grabbed from ipa10-nrh or OKD
+PROXSTAR_LDAP_BIND_PW= # Ditto
+
+# DB
+PROXSTAR_SQLALCHEMY_DATABASE_URI=postgresql://postgres:changeme@proxstar-postgres/proxstar
+
+# REDIS
+PROXSTAR_REDIS_HOST=proxstar-redis
+PROXSTAR_REDIS_PORT=6379
+
+# VNC
+PROXSTAR_WEBSOCKIFY_PATH=/opt/app-root/bin/websockify
+PROXSTAR_WEBSOCKIFY_TARGET_FILE=/opt/app-root/src/targets
+
+# SENTRY
+# If you set the sentry dsn locally, make sure you use the local-dev or some
+# other local environment, so we can separate local errors from production
+PROXSTAR_SENTRY_DSN=
+PROXSTAR_SENTRY_ENV=local-dev
+
+# DATADOG RUM
+PROXSTAR_DD_CLIENT_TOKEN=
+PROXSTAR_DD_APP_ID=
+
+# GUNICORN
+PROXSTAR_TIMEOUT=120

HACKING/.gitignore

@@ -0,0 +1,3 @@
+volume/
+volume/*
+.env

HACKING/README.md

@@ -0,0 +1,48 @@
+## Setting up a full dev environment
+
+If you want to work on Proxstar using a 1:1 development setup, there are a couple things you're going to need
+
+- A machine you can
+    - SSH into
+        - With portforwarding (see `man ssh` for info on the `-L` option)
+    - and run
+        - Flask
+        - Redis
+        - Docker
+- At least one (1) Proxmox host running Proxmox >6.3
+- A CSH account
+- An RTP (to tell you secrets)
+
+1. Configure your Proxmox node (Not required if you're using the CSH cluster)
+
+I would recommend setting up a development account on your Proxmox node. Name it anything. (Maybe `proxstartest`?). This is necessary to grab authentication tokens and the like. It should have the same permissions as `root@pam`. You can accomplish this by creating a group in `Datacenter > Permissions > Groups` and adding `Administrator` permissions to the group, then adding your user to the group. If you do this, then it's easy to enable/disable it for development. You should also generate an SSH key for the user.
+
+When you log into your Proxstar instance, it should auto-create the pool. If for some reason it doesn't, you can set up a pool on your Proxmox node with your CSH username. To do this, go into `Datacenter > Permissions > Pools > Create`.
+
+2. Set up your environment
+
+If you're trying to run this all on a VM without a graphical web browser, you can forward traffic to your computer using SSH.
+```
+ssh example@dev-server.csh.rit.edu -L 8000:localhost:8000
+```
+# New Deployment Instructions
+
+1.  Build your containers. The `proxstar` container serves as proxstar, rq, rq-scheduler, and VNC. The `proxstar-postgres` container sets up the database schema.
+
+`mkdir HACKING/proxstar-postgres/volume`
+
+`podman build . --tag=proxstar`
+
+`podman build HACKING/proxstar-postgres --tag=proxstar-postgres`
+
+2. Configure your environment variables. I'd recommend setting up a .env file and passing that into your container. Check `.env.template` for more info.
+
+3. Run it. This sets up redis, postgres, rq, and proxstar.
+
+```
+podman run --rm -d --network=proxstar --name=proxstar-redis redis:alpine
+podman run --rm -d --network=proxstar --name=proxstar-postgres -e POSTGRES_PASSWORD=changeme -v ./HACKING/proxstar-postgres/volume:/var/lib/postgresql/data:Z proxstar-postgres
+podman run --rm -d --network=proxstar --name=proxstar-rq-scheduler  --env-file=HACKING/.env --entrypoint ./start_scheduler.sh proxstar
+podman run --rm -d --network=proxstar --name=proxstar-rq  --env-file=HACKING/.env --entrypoint ./start_worker.sh proxstar
+podman run --rm -d --network=proxstar --name=proxstar -p 8000:8000 --env-file=HACKING/.env proxstar
+```

HACKING/proxstar-postgres/Dockerfile

@@ -6,6 +6,4 @@ RUN apt-get update \
     && cpanm Data::Validate::Domain \
     && apt-get remove -y build-essential
 
-#COPY ./schema/large.sql /docker-entrypoint-initdb.d/ # Woa there, pal; That's a lot of   S C H E M A.
+COPY ./schema/large.sql /docker-entrypoint-initdb.d/
-# I volunteer as tribute. DM me if this breaks.
-RUN curl https://csh.rit.edu/~wilnil/proxstar-postgres/large.sql -o /docker-entrypoint-initdb.d/large.sql

developing/README.md

@@ -1,199 +0,0 @@
-## Setting up a full dev environment
-
-If you want to work on Proxstar using a 1:1 development setup, there are a couple things you're going to need
-
-- A machine you can
-    - SSH into
-        - With portforwarding (see `man ssh` for info on the `-L` option)
-    - and run
-        - Flask
-        - Redis
-        - Docker
-- At least one (1) Proxmox host running Proxmox >6.3
-- A CSH account
-- An RTP (to tell you secrets)
-
-1. Configure your Proxmox node
-
-I would recommend setting up a development account on your Proxmox node. Name it anything. (Maybe `proxstartest`?). This is necessary to grab authentication tokens and the like. It should have the same permissions as `root@pam`. You can accomplish this by creating a group in `Datacenter > Permissions > Groups` and adding `Administrator` permissions to the group, then adding your user to the group. If you do this, then it's easy to enable/disable it for development. You should also generate an SSH key for the user.
-
-When you log into your Proxstar instance, it should auto-create the pool. If for some reason it doesn't, you can set up a pool on your Proxmox node with your CSH username. To do this, go into `Datacenter > Permissions > Pools > Create`.
-
-2. Set up your environment
-
-If you're trying to run this all on a VM without a graphical web browser, you can forward traffic to your computer using SSH.
-```
-ssh example@dev-server.csh.rit.edu -L 8000:localhost:8000
-```
-
-Clone down the repository, and create a Virtualenv to do your work in.
-```
-mkdir venv
-python3.8 -m venv venv
-source venv/bin/activate
-```
-
-Install required Python modules
-```
-pip install -r requirements.txt
-pip install click==7.1.2 python-dotenv
-```
-Fill out the required fields in your config_local.py file. You might have to come back to this after you run the docker compose.
-```
-cp config.py config_local.py
-vim config_local.py
-```
-
-(Here's some advice on how to fill out your config file.)
-```
-  from os import environ
-
-  # Proxstar
-  VM_EXPIRE_MONTHS = int(environ.get('PROXSTAR_VM_EXPIRE_MONTHS', '3'))
-  VNC_CLEANUP_TOKEN = environ.get('PROXSTAR_VNC_CLEANUP_TOKEN', '')
-
-  # Flask
-
-  # The IP address to which Proxstar is served.
-  # You should probably set this to 127.0.0.1 if you're developing on your
-  # local machine, or portforwarding through SSH.
-  # 0.0.0.0 will serve to wherever you want. Don't do that unless you know
-  # what you're doing.
-  IP = environ.get('PROXSTAR_IP', '127.0.0.1') 
-
-  # The port Proxstar runs on.
-  # Because sso is configured to accept from `http://localhost:8000', you should
-  # set this to 8000 for development.
-  PORT = environ.get('PROXSTAR_PORT', '5000')
-
-  # The name of your proxstar server. This matters for authenticating with CSH
-  # SSO, so change this to localhost:8000
-  SERVER_NAME = environ.get('PROXSTAR_SERVER_NAME', 'proxstar.csh.rit.edu')
-
-  # Secret key for authenticating with SSO.
-  # Change this to literally anything, just don't leave it blank.
-  SECRET_KEY = environ.get('PROXSTAR_SECRET_KEY', '')
-
-  # OIDC
-
-  # Leave all of this alone.
-  OIDC_ISSUER = environ.get('PROXSTAR_OIDC_ISSUER', 'https://sso.csh.rit.edu/auth/realms/csh')
-  OIDC_CLIENT_CONFIG = {
-      'client_id': environ.get('PROXSTAR_CLIENT_ID', 'proxstar'),
-      'client_secret': environ.get('PROXSTAR_CLIENT_SECRET', ''), # Just kidding, talk to an RTP to get this.
-      'post_logout_redirect_uris': [
-          environ.get('PROXSTAR_REDIRECT_URI', 'https://proxstar.csh.rit.edu/logout')
-      ],
-  }
-
-  # Proxmox
-
-  # Your list of proxmox hosts. You only need one for development.
-  PROXMOX_HOSTS = [host.strip() for host in environ.get('PROXSTAR_PROXMOX_HOSTS', '').split(',')]
-  # Username and group of your test user. For example, 'proxstartest@pam'
-  PROXMOX_USER = environ.get('PROXSTAR_PROXMOX_USER', '')
-  # Said user's password
-  PROXMOX_PASS = environ.get('PROXSTAR_PROXMOX_PASS', '')
-  # Location of ISO storage on your server. CSH has an NFS share for this,
-  # but usually this is 'local'
-  PROXMOX_ISO_STORAGE = environ.get('PROXSTAR_PROXMOX_ISO_STORAGE', 'local')
-  # Location of storage for VMs. CSH has a ceph cluster, but change this to
-  # whatever the name of your cluster's storage is. By default Proxmox uses
-  # local-lvm
-  PROXMOX_VM_STORAGE = environ.get('PROXSTAR_PROXMOX_VM_STORAGE', 'ceph')
-  # Username of SSH user (probably the same)
-  PROXMOX_SSH_USER = environ.get('PROXSTAR_PROXMOX_SSH_USER', '')
-  # Paste that SSH key I told you to generate.
-  PROXMOX_SSH_KEY = environ.get('PROXSTAR_PROXMOX_SSH_KEY', '')
-  # If you put a password on it, then paste that here.
-  PROXMOX_SSH_KEY_PASS = environ.get('PROXSTAR_PROXMOX_SSH_KEY_PASS', '')
-
-  # STARRS
-
-  # The IP address or hostname of your STARRs host.
-
-  # Since you should be hosting this in a container, make it 127.0.0.1
-  STARRS_DB_HOST = environ.get('PROXSTAR_STARRS_DB_HOST', '')
-
-  # The name of your STARRS DB
-  # It.... it should be STARRS.
-  STARRS_DB_NAME = environ.get('PROXSTAR_DB_NAME', 'starrs')
-
-  # The username of your STARRS DB
-  # I just used the postgres user and it seemed to work so uhhhhhhhhhhh
-  STARRS_DB_USER = environ.get('PROXSTAR_DB_USER', '')
-
-  # Password for Postgres user
-  # (You configure this when setting up the Postgres container just use that PWord)
-  STARRS_DB_PASS = environ.get('PROXSTAR_DB_PASS', '')
-
-  # STARRS username
-  # Leave this alone.
-  STARRS_USER = environ.get('PROXSTAR_STARRS_USER', 'proxstar')
-
-  #???
-  # IDK leave this alone, too.
-  STARRS_IP_RANGE = environ.get('PROXSTAR_IP_RANGE', '')
-
-  # LDAP
-
-  # You can just use your LDAP Bind DN and Password here
-  # (remember to keep them hidden!)
-  LDAP_BIND_DN = environ.get('PROXSTAR_LDAP_BIND_DN', '')
-  LDAP_BIND_PW = environ.get('PROXSTAR_LDAP_BIND_PW', '')
-
-  # DB
-
-  # The URI to your proxstar DB.
-  # Probably looks like this: postgresql://postgres:********@localhost/proxstar
-  SQLALCHEMY_DATABASE_URI = environ.get('PROXSTAR_SQLALCHEMY_DATABASE_URI', '')
-
-  # REDIS
-  # Leave this alone. This will point at your Redis container.
-  REDIS_HOST = environ.get('PROXSTAR_REDIS_HOST', 'localhost')
-  RQ_DASHBOARD_REDIS_HOST = environ.get('PROXSTAR_REDIS_HOST', 'localhost')
-  REDIS_PORT = int(environ.get('PROXSTAR_REDIS_PORT', '6379'))
-
-  # VNC
-
-  #Haha this is so fucking busted. Leave this alone.
-  WEBSOCKIFY_PATH = environ.get('PROXSTAR_WEBSOCKIFY_PATH', '/opt/app-root/bin/websockify')
-  WEBSOCKIFY_TARGET_FILE = environ.get('PROXSTAR_WEBSOCKIFY_TARGET_FILE', '/opt/app-root/src/targets')
-
-  # SENTRY
-  # If you set the sentry dsn locally, make sure you use the local-dev or some
-  # other local environment, so we can separate local errors from production
-
-  # Leave this alone, too.
-  SENTRY_DSN = environ.get('PROXSTAR_SENTRY_DSN', '')
-  RQ_SENTRY_DSN = environ.get('PROXSTAR_SENTRY_DSN', '')
-  SENTRY_ENV = environ.get('PROXSTAR_SENTRY_ENV', 'local-dev')
-
-  # DATADOG RUM
-
-  # Leave this alone, too
-  DD_CLIENT_TOKEN = environ.get('PROXSTAR_DD_CLIENT_TOKEN', '')
-  DD_APP_ID = environ.get('PROXSTAR_DD_APP_ID', '')
-
-  # GUNICORN
-
-  # Yeah whatever, leave it alone.
-  TIMEOUT = environ.get('PROXSTAR_TIMEOUT', 120)
-```
-
-Now, go ahead and run the Docker Compose file to set up your Postgres and Redis instances.
-
-```
-docker-compose up -d
-```
-
-Now, you should be ready to run your dev instance. I like to use `tmux` for this to run proxstar and the `rq worker` in separate panes.
-
-```
-  flask run -p 8000 --cert=adhoc
-  rq worker
-```
-
-(You might have to specify your host as `-h 127.0.0.1` if Flask is misbehaving)
-
-Open a web browser and navigate to http://localhost:8000. You should see Proxstar running.

developing/docker-compose.yml

@@ -1,20 +0,0 @@
-version: "3.9"
-services:
-  proxstar-redis:
-    image: "redis:alpine"
-    ports:
-      - 127.0.0.1:6379:6379
-  proxstar-postgres:
-    build: ./proxstar-postgres
-    # image: postgres
-    # restart: always
-    ports:
-      - 127.0.0.1:5432:5432
-    environment:
-      POSTGRES_PASSWORD: tits12348
-    volumes:
-      - type: volume
-        source: proxstar-postgres
-        target: /var/lib/postgresql/data
-volumes:
-  proxstar-postgres:

requirements.txt

@@ -1,7 +1,7 @@
 black~=21.9b0
-csh-ldap~=2.3.1
+csh-ldap==2.4.0
 click~=7.1.2
-ddtrace~=1.1.4
+ddtrace~=1.2.1
 flask==1.1.4
 jinja2==2.11.3
 flask-pyoidc==1.3.0
@@ -14,7 +14,7 @@ psycopg2-binary==2.9.3
 python-dateutil==2.8.1
 redis==3.5.3
 requests==2.25.1
-rq==1.1.0
+rq==1.10.1
 rq-dashboard==0.6.1
 rq-scheduler==0.10.0
 sqlalchemy==1.3.22
@@ -25,3 +25,4 @@ pylint==2.13.9
 pylint-quotes==0.2.3
 sentry-sdk[flask]
 sentry-sdk~=1.5.12
+python-dotenv==0.19.1

start_scheduler.sh

@@ -1,3 +1,5 @@
 #!/bin/sh
 
-/opt/app-root/bin/rqscheduler -u "$PROXSTAR_REDIS_URL"
+PROXSTAR_REDIS_URL=redis://$PROXSTAR_REDIS_HOST:$PROXSTAR_REDIS_PORT
+
+rqscheduler -u "$PROXSTAR_REDIS_URL"

start_worker.sh

@@ -1,3 +1,5 @@
 #!/bin/sh
 
-/opt/app-root/bin/rq worker -u "$PROXSTAR_REDIS_URL" --sentry-dsn "" -c rqsettings
+PROXSTAR_REDIS_URL=redis://$PROXSTAR_REDIS_HOST:$PROXSTAR_REDIS_PORT
+
+rq worker -u "$PROXSTAR_REDIS_URL" --sentry-dsn "" -c rqsettings