diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..790cb47 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.8-buster +WORKDIR /opt/proxstar +RUN apt-get update -y && apt-get install -y python3-dev libldap2-dev libsasl2-dev ldap-utils +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY start_worker.sh start_scheduler.sh . +COPY .git ./.git +COPY *.py . +COPY proxstar ./proxstar +RUN touch proxmox_ssh_key && chmod a+w proxmox_ssh_key # This is some OKD shit. +ENTRYPOINT python3 wsgi.py \ No newline at end of file diff --git a/HACKING/.env.template b/HACKING/.env.template new file mode 100644 index 0000000..0e0371a --- /dev/null +++ b/HACKING/.env.template @@ -0,0 +1,67 @@ +# Proxstar +PROXSTAR_VM_EXPIRE_MONTHS=3 +PROXSTAR_VNC_CLEANUP_TOKEN= + +# Development options +# Determines weather or not to run STARRS queries (for doing stuff like checking for available IPs) +PROXSTAR_USE_STARRS=True +# If you're an RTP and want to see a normal user's homepage view, set this to True. +PROXSTAR_FORCE_STANDARD_USER=False + +# Flask +PROXSTAR_IP=0.0.0.0 # The IP address to which Proxstar is served. +PROXSTAR_PORT=8000 # The port Proxstar runs on. +PROXSTAR_SERVER_NAME=localhost:8000 # It's configured this way in SSO. +PROXSTAR_SECRET_KEY= # This can be LITERALLY anything. + +# OIDC +PROXSTAR_OIDC_ISSUER=https://sso.csh.rit.edu/auth/realms/csh +PROXSTAR_CLIENT_ID= # Ask an RTP. Can get this from OKD configs. +PROXSTAR_CLIENT_SECRET= # Ask an RTP. +PROXSTAR_REDIRECT_URI=https://proxstar.csh.rit.edu/logout + +# Proxmox +PROXSTAR_PROXMOX_HOSTS= # Host list that Proxstar can use (proxstar01, etc...) +PROXSTAR_PROXMOX_USER=api@pve +PROXSTAR_PROXMOX_PASS= # Password for proxstar user +PROXSTAR_PROXMOX_ISO_STORAGE=nfs-iso +PROXSTAR_PROXMOX_VM_STORAGE=ceph +PROXSTAR_PROXMOX_SSH_USER=root +PROXSTAR_PROXMOX_SSH_KEY="" # Ask an RTP. This is gonna look like a certificate. +PROXSTAR_PROXMOX_SSH_KEY_PASS= # Password for above certificate + +# STARRS +PROXSTAR_STARRS_DB_HOST=proxstar-postgres +PROXSTAR_DB_NAME=starrs +PROXSTAR_DB_USER=postgres +PROXSTAR_DB_PASS=changeme +PROXSTAR_STARRS_USER=proxstar +PROXSTAR_IP_RANGE= + +# LDAP +PROXSTAR_LDAP_BIND_DN= # Ask an RTP. Can be grabbed from ipa10-nrh or OKD +PROXSTAR_LDAP_BIND_PW= # Ditto + +# DB +PROXSTAR_SQLALCHEMY_DATABASE_URI=postgresql://postgres:changeme@proxstar-postgres/proxstar + +# REDIS +PROXSTAR_REDIS_HOST=proxstar-redis +PROXSTAR_REDIS_PORT=6379 + +# VNC +PROXSTAR_WEBSOCKIFY_PATH=/opt/app-root/bin/websockify +PROXSTAR_WEBSOCKIFY_TARGET_FILE=/opt/app-root/src/targets + +# SENTRY +# If you set the sentry dsn locally, make sure you use the local-dev or some +# other local environment, so we can separate local errors from production +PROXSTAR_SENTRY_DSN= +PROXSTAR_SENTRY_ENV=local-dev + +# DATADOG RUM +PROXSTAR_DD_CLIENT_TOKEN= +PROXSTAR_DD_APP_ID= + +# GUNICORN +PROXSTAR_TIMEOUT=120 diff --git a/HACKING/.gitignore b/HACKING/.gitignore new file mode 100644 index 0000000..1926e18 --- /dev/null +++ b/HACKING/.gitignore @@ -0,0 +1,3 @@ +volume/ +volume/* +.env \ No newline at end of file diff --git a/HACKING/README.md b/HACKING/README.md index cb68576..d893f4d 100644 --- a/HACKING/README.md +++ b/HACKING/README.md @@ -13,7 +13,7 @@ If you want to work on Proxstar using a 1:1 development setup, there are a coupl - A CSH account - An RTP (to tell you secrets) -1. Configure your Proxmox node +1. Configure your Proxmox node (Not required if you're using the CSH cluster) I would recommend setting up a development account on your Proxmox node. Name it anything. (Maybe `proxstartest`?). This is necessary to grab authentication tokens and the like. It should have the same permissions as `root@pam`. You can accomplish this by creating a group in `Datacenter > Permissions > Groups` and adding `Administrator` permissions to the group, then adding your user to the group. If you do this, then it's easy to enable/disable it for development. You should also generate an SSH key for the user. @@ -25,180 +25,22 @@ If you're trying to run this all on a VM without a graphical web browser, you ca ``` ssh example@dev-server.csh.rit.edu -L 8000:localhost:8000 ``` +# New Deployment Instructions -Clone down the repository, and create a Virtualenv to do your work in. -``` -mkdir venv -python3.8 -m venv venv -source venv/bin/activate -``` +1. Build your containers. The `proxstar` container serves as proxstar, rq, rq-scheduler, and VNC. The `proxstar-postgres` container sets up the database schema. -Install required packages -``` -dnf install python3-devel -``` +`podman build . --tag=proxstar` -Install required Python modules -``` -pip install -r requirements.txt -pip install click==7.1.2 python-dotenv -``` -Fill out the required fields in your config_local.py file. You might have to come back to this after you run the docker compose. -``` -cp config.py config_local.py -vim config_local.py -``` +`podman build HACKING/proxstar-postgres --tag=proxstar-postgres` -(Here's some advice on how to fill out your config file.) -``` - from os import environ +2. Configure your environment variables. I'd recommend setting up a .env file and passing that into your container. Check `.env.template` for more info. - # Proxstar - VM_EXPIRE_MONTHS = int(environ.get('PROXSTAR_VM_EXPIRE_MONTHS', '3')) - VNC_CLEANUP_TOKEN = environ.get('PROXSTAR_VNC_CLEANUP_TOKEN', '') - - # Flask - - # The IP address to which Proxstar is served. - # You should probably set this to 127.0.0.1 if you're developing on your - # local machine, or portforwarding through SSH. - # 0.0.0.0 will serve to wherever you want. Don't do that unless you know - # what you're doing. - IP = environ.get('PROXSTAR_IP', '127.0.0.1') - - # The port Proxstar runs on. - # Because sso is configured to accept from `http://localhost:8000', you should - # set this to 8000 for development. - PORT = environ.get('PROXSTAR_PORT', '5000') - - # The name of your proxstar server. This matters for authenticating with CSH - # SSO, so change this to localhost:8000 - SERVER_NAME = environ.get('PROXSTAR_SERVER_NAME', 'proxstar.csh.rit.edu') - - # Secret key for authenticating with SSO. - # Change this to literally anything, just don't leave it blank. - SECRET_KEY = environ.get('PROXSTAR_SECRET_KEY', '') - - # OIDC - - # Leave all of this alone. - OIDC_ISSUER = environ.get('PROXSTAR_OIDC_ISSUER', 'https://sso.csh.rit.edu/auth/realms/csh') - OIDC_CLIENT_CONFIG = { - 'client_id': environ.get('PROXSTAR_CLIENT_ID', 'proxstar'), - 'client_secret': environ.get('PROXSTAR_CLIENT_SECRET', ''), # Just kidding, talk to an RTP to get this. - 'post_logout_redirect_uris': [ - environ.get('PROXSTAR_REDIRECT_URI', 'https://proxstar.csh.rit.edu/logout') - ], - } - - # Proxmox - - # Your list of proxmox hosts. You only need one for development. - PROXMOX_HOSTS = [host.strip() for host in environ.get('PROXSTAR_PROXMOX_HOSTS', '').split(',')] - # Username and group of your test user. For example, 'proxstartest@pam' - PROXMOX_USER = environ.get('PROXSTAR_PROXMOX_USER', '') - # Said user's password - PROXMOX_PASS = environ.get('PROXSTAR_PROXMOX_PASS', '') - # Location of ISO storage on your server. CSH has an NFS share for this, - # but usually this is 'local' - PROXMOX_ISO_STORAGE = environ.get('PROXSTAR_PROXMOX_ISO_STORAGE', 'local') - # Location of storage for VMs. CSH has a ceph cluster, but change this to - # whatever the name of your cluster's storage is. By default Proxmox uses - # local-lvm - PROXMOX_VM_STORAGE = environ.get('PROXSTAR_PROXMOX_VM_STORAGE', 'ceph') - # Username of SSH user (probably the same) - PROXMOX_SSH_USER = environ.get('PROXSTAR_PROXMOX_SSH_USER', '') - # Paste that SSH key I told you to generate. - PROXMOX_SSH_KEY = environ.get('PROXSTAR_PROXMOX_SSH_KEY', '') - # If you put a password on it, then paste that here. - PROXMOX_SSH_KEY_PASS = environ.get('PROXSTAR_PROXMOX_SSH_KEY_PASS', '') - - # STARRS - - # The IP address or hostname of your STARRs host. - - # Since you should be hosting this in a container, make it 127.0.0.1 - STARRS_DB_HOST = environ.get('PROXSTAR_STARRS_DB_HOST', '') - - # The name of your STARRS DB - # It.... it should be STARRS. - STARRS_DB_NAME = environ.get('PROXSTAR_DB_NAME', 'starrs') - - # The username of your STARRS DB - # I just used the postgres user and it seemed to work so uhhhhhhhhhhh - STARRS_DB_USER = environ.get('PROXSTAR_DB_USER', '') - - # Password for Postgres user - # (You configure this when setting up the Postgres container just use that PWord) - STARRS_DB_PASS = environ.get('PROXSTAR_DB_PASS', '') - - # STARRS username - # Leave this alone. - STARRS_USER = environ.get('PROXSTAR_STARRS_USER', 'proxstar') - - #??? - # IDK leave this alone, too. - STARRS_IP_RANGE = environ.get('PROXSTAR_IP_RANGE', '') - - # LDAP - - # You can just use your LDAP Bind DN and Password here - # (remember to keep them hidden!) - LDAP_BIND_DN = environ.get('PROXSTAR_LDAP_BIND_DN', '') - LDAP_BIND_PW = environ.get('PROXSTAR_LDAP_BIND_PW', '') - - # DB - - # The URI to your proxstar DB. - # Probably looks like this: postgresql://postgres:********@localhost/proxstar - SQLALCHEMY_DATABASE_URI = environ.get('PROXSTAR_SQLALCHEMY_DATABASE_URI', '') - - # REDIS - # Leave this alone. This will point at your Redis container. - REDIS_HOST = environ.get('PROXSTAR_REDIS_HOST', 'localhost') - RQ_DASHBOARD_REDIS_HOST = environ.get('PROXSTAR_REDIS_HOST', 'localhost') - REDIS_PORT = int(environ.get('PROXSTAR_REDIS_PORT', '6379')) - - # VNC - - #Haha this is so fucking busted. Leave this alone. - WEBSOCKIFY_PATH = environ.get('PROXSTAR_WEBSOCKIFY_PATH', '/opt/app-root/bin/websockify') - WEBSOCKIFY_TARGET_FILE = environ.get('PROXSTAR_WEBSOCKIFY_TARGET_FILE', '/opt/app-root/src/targets') - - # SENTRY - # If you set the sentry dsn locally, make sure you use the local-dev or some - # other local environment, so we can separate local errors from production - - # Leave this alone, too. - SENTRY_DSN = environ.get('PROXSTAR_SENTRY_DSN', '') - RQ_SENTRY_DSN = environ.get('PROXSTAR_SENTRY_DSN', '') - SENTRY_ENV = environ.get('PROXSTAR_SENTRY_ENV', 'local-dev') - - # DATADOG RUM - - # Leave this alone, too - DD_CLIENT_TOKEN = environ.get('PROXSTAR_DD_CLIENT_TOKEN', '') - DD_APP_ID = environ.get('PROXSTAR_DD_APP_ID', '') - - # GUNICORN - - # Yeah whatever, leave it alone. - TIMEOUT = environ.get('PROXSTAR_TIMEOUT', 120) -``` - -Now, go ahead and run the Docker Compose file to set up your Postgres and Redis instances. +3. Run it with this clusterfuck. This sets up redis, postgres, rq, and proxstar. ``` -docker-compose up -d +podman run --rm -d --network=proxstar --name=proxstar-redis redis:alpine +podman run --rm -d --network=proxstar --name=proxstar-postgres -e POSTGRES_PASSWORD=changeme -v ./HACKING/proxstar-postgres//volume:/var/lib/postgresql/data:Z proxstar-postgres +podman run --rm -d --network=proxstar --name=proxstar-rq-scheduler --env-file=HACKING/.env --entrypoint ./start_scheduler.sh proxstar +podman run --rm -d --network=proxstar --name=proxstar-rq --env-file=HACKING/.env --entrypoint ./start_worker.sh proxstar +podman run --rm -d --network=proxstar --name=proxstar -p 8000:8000 --env-file=HACKING/.env proxstar ``` - -Now, you should be ready to run your dev instance. I like to use `tmux` for this to run proxstar and the `rq worker` in separate panes. - -``` - flask run -p 8000 --cert=adhoc - rq worker -``` - -(You might have to specify your host as `-h 127.0.0.1` if Flask is misbehaving) - -Open a web browser and navigate to http://localhost:8000. You should see Proxstar running. diff --git a/HACKING/docker-compose.yml b/HACKING/docker-compose.yml deleted file mode 100644 index cebacb1..0000000 --- a/HACKING/docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: "3.9" -services: - proxstar-redis: - image: "redis:alpine" - ports: - - 127.0.0.1:6379:6379 - proxstar-postgres: - build: ./proxstar-postgres - # image: postgres - # restart: always - ports: - - 127.0.0.1:5432:5432 - environment: - POSTGRES_PASSWORD: tits12348 - volumes: - - type: volume - source: proxstar-postgres - target: /var/lib/postgresql/data -volumes: - proxstar-postgres: diff --git a/HACKING/proxstar-postgres/Dockerfile b/HACKING/proxstar-postgres/Dockerfile index 6693205..1243e3f 100644 --- a/HACKING/proxstar-postgres/Dockerfile +++ b/HACKING/proxstar-postgres/Dockerfile @@ -8,4 +8,4 @@ RUN apt-get update \ #COPY ./schema/large.sql /docker-entrypoint-initdb.d/ # Woa there, pal; That's a lot of S C H E M A. # I volunteer as tribute. DM me if this breaks. -RUN curl https://csh.rit.edu/~wilnil/proxstar-postgres/large.sql -o /docker-entrypoint-initdb.d/large.sql +RUN curl https://csh.rit.edu/~wilnil/storage/proxstar-postgres/large.sql -o /docker-entrypoint-initdb.d/large.sql diff --git a/requirements.txt b/requirements.txt index d952a15..6070785 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ black~=21.9b0 -csh-ldap~=2.3.1 +csh-ldap==2.4.0 click~=7.1.2 ddtrace~=1.1.4 flask==1.1.4 @@ -14,7 +14,7 @@ psycopg2-binary==2.9.3 python-dateutil==2.8.1 redis==3.5.3 requests==2.25.1 -rq==1.1.0 +rq==1.10.1 rq-dashboard==0.6.1 rq-scheduler==0.10.0 sqlalchemy==1.3.22 @@ -25,4 +25,4 @@ pylint==2.13.9 pylint-quotes==0.2.3 sentry-sdk[flask] sentry-sdk~=1.5.12 -python-dotenv==0.19.1 \ No newline at end of file +python-dotenv==0.19.1 diff --git a/start_scheduler.sh b/start_scheduler.sh index e95f14c..9b767f0 100755 --- a/start_scheduler.sh +++ b/start_scheduler.sh @@ -1,3 +1,5 @@ #!/bin/sh -/opt/app-root/bin/rqscheduler -u "$PROXSTAR_REDIS_URL" +PROXSTAR_REDIS_URL=redis://$PROXSTAR_REDIS_HOST:$PROXSTAR_REDIS_PORT + +rqscheduler -u "$PROXSTAR_REDIS_URL" diff --git a/start_worker.sh b/start_worker.sh index 2f609f3..ab291ff 100755 --- a/start_worker.sh +++ b/start_worker.sh @@ -1,3 +1,5 @@ #!/bin/sh -/opt/app-root/bin/rq worker -u "$PROXSTAR_REDIS_URL" --sentry-dsn "" -c rqsettings +PROXSTAR_REDIS_URL=redis://$PROXSTAR_REDIS_HOST:$PROXSTAR_REDIS_PORT + +rq worker -u "$PROXSTAR_REDIS_URL" --sentry-dsn "" -c rqsettings