From c4c51106fa58a01c485eb3b1f585a6818296e971 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 17 Apr 2021 00:05:18 +0100 Subject: [PATCH 01/11] get nginx to proxy websockets correctly --- .../templates/jupyterhub-nginx.conf | 42 +++++++++++-------- roles/nginx/templates/nginx.conf.j2 | 9 ++++ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/roles/jupyterhub/templates/jupyterhub-nginx.conf b/roles/jupyterhub/templates/jupyterhub-nginx.conf index 301ab2a41..6758458e9 100644 --- a/roles/jupyterhub/templates/jupyterhub-nginx.conf +++ b/roles/jupyterhub/templates/jupyterhub-nginx.conf @@ -1,20 +1,26 @@ -location /jupyterhub { - proxy_pass http://127.0.0.1:8000; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-NginX-Proxy true; +location /jupyterhub { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + } - # websocket headers - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - #proxy_set_header Connection $connection_upgrade; - proxy_set_header X-Scheme $scheme; +# Managing WebHook/Socket requests between hub user servers and external proxy + location ~* /(api/kernels/[^/]+/(channels|iopub|shell|stdin)|terminals/websocket)/? { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_buffering off; -} - -# Managing requests to verify letsencrypt host -location ~ /.well-known { - allow all; -} + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + #proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Scheme $scheme; + proxy_buffering off; + } + # Managing requests to verify letsencrypt host + location ~ /.well-known { + allow all; + } diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2 index 10f33f72f..10fc4cfda 100644 --- a/roles/nginx/templates/nginx.conf.j2 +++ b/roles/nginx/templates/nginx.conf.j2 @@ -67,6 +67,15 @@ http { # gzip_buffers 16 8k; # gzip_http_version 1.1; # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + # top-level http config for websocket headers + # If Upgrade is defined, Connection = upgrade + # If Upgrade is empty, Connection = close + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + ## # Virtual Host Configs ## From fae7a07368e06378c1af0644365dde165dae092f Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 17 Apr 2021 05:02:00 +0100 Subject: [PATCH 02/11] uncomment websocket upgrade line --- roles/jupyterhub/templates/jupyterhub-nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/jupyterhub/templates/jupyterhub-nginx.conf b/roles/jupyterhub/templates/jupyterhub-nginx.conf index 6758458e9..2911a5d5a 100644 --- a/roles/jupyterhub/templates/jupyterhub-nginx.conf +++ b/roles/jupyterhub/templates/jupyterhub-nginx.conf @@ -16,7 +16,7 @@ location /jupyterhub { # WebSocket support proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; - #proxy_set_header Connection $connection_upgrade; + proxy_set_header Connection $connection_upgrade; proxy_set_header X-Scheme $scheme; proxy_buffering off; } From 77cf77d07458cc6210bbe14d6f5e803142ab00dd Mon Sep 17 00:00:00 2001 From: root Date: Sat, 17 Apr 2021 11:43:04 +0000 Subject: [PATCH 03/11] Lint jupyterhub-nginx.conf & nginx.conf etc, for maintainability --- roles/jupyterhub/tasks/enable-or-disable.yml | 2 +- roles/jupyterhub/tasks/install.yml | 16 +-- .../templates/jupyterhub-nginx.conf | 47 +++---- roles/nginx/templates/nginx.conf.j2 | 125 +++++++++--------- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/roles/jupyterhub/tasks/enable-or-disable.yml b/roles/jupyterhub/tasks/enable-or-disable.yml index 1c8a720cc..8aa7c0e57 100644 --- a/roles/jupyterhub/tasks/enable-or-disable.yml +++ b/roles/jupyterhub/tasks/enable-or-disable.yml @@ -18,7 +18,7 @@ when: not jupyterhub_enabled -- name: 'Install from template: {{ nginx_conf_dir }}/jupyterhub-nginx.conf' +- name: "Install from template: {{ nginx_conf_dir }}/jupyterhub-nginx.conf" template: src: jupyterhub-nginx.conf dest: "{{ nginx_conf_dir }}/" diff --git a/roles/jupyterhub/tasks/install.yml b/roles/jupyterhub/tasks/install.yml index 26ea0cb01..8f35b418c 100644 --- a/roles/jupyterhub/tasks/install.yml +++ b/roles/jupyterhub/tasks/install.yml @@ -16,11 +16,11 @@ - name: Make the directories to hold JupyterHub config file: state: directory - path: '{{ item }}' + path: "{{ item }}" with_items: - - '{{ jupyterhub_venv }}/etc/jupyter' - - '{{ jupyterhub_venv }}/etc/jupyterhub' - - '{{ jupyterhub_venv }}/etc/systemd' + - "{{ jupyterhub_venv }}/etc/jupyter" + - "{{ jupyterhub_venv }}/etc/jupyterhub" + - "{{ jupyterhub_venv }}/etc/systemd" - name: Use npm to install configurable-http-proxy npm: @@ -28,7 +28,7 @@ global: yes state: latest -- name: 'Use pip to install into a virtual environment: {{ jupyterhub_venv }}' +- name: "Use pip to install into a virtual environment: {{ jupyterhub_venv }}" pip: name: - pip @@ -45,12 +45,12 @@ extra_args: "--no-cache-dir" when: internet_available -- name: 'Install from template: {{ jupyterhub_venv }}/etc/jupyterhub/jupyterhub_config.py' +- name: "Install from template: {{ jupyterhub_venv }}/etc/jupyterhub/jupyterhub_config.py" template: src: jupyterhub_config.py - dest: '{{ jupyterhub_venv }}/etc/jupyterhub/' + dest: "{{ jupyterhub_venv }}/etc/jupyterhub/" -- name: 'Install from template: /etc/systemd/system/jupyterhub.service' +- name: "Install from template: /etc/systemd/system/jupyterhub.service" template: src: jupyterhub.service dest: /etc/systemd/system/ diff --git a/roles/jupyterhub/templates/jupyterhub-nginx.conf b/roles/jupyterhub/templates/jupyterhub-nginx.conf index 2911a5d5a..6d098f861 100644 --- a/roles/jupyterhub/templates/jupyterhub-nginx.conf +++ b/roles/jupyterhub/templates/jupyterhub-nginx.conf @@ -1,26 +1,27 @@ -location /jupyterhub { - proxy_pass http://127.0.0.1:8000; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-NginX-Proxy true; - } +location /jupyterhub { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; +} # Managing WebHook/Socket requests between hub user servers and external proxy - location ~* /(api/kernels/[^/]+/(channels|iopub|shell|stdin)|terminals/websocket)/? { - proxy_pass http://127.0.0.1:8000; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +location ~* /(api/kernels/[^/]+/(channels|iopub|shell|stdin)|terminals/websocket)/? { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # WebSocket support - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header X-Scheme $scheme; - proxy_buffering off; - } - # Managing requests to verify letsencrypt host - location ~ /.well-known { - allow all; - } + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Scheme $scheme; + proxy_buffering off; +} + +# Managing requests to verify letsencrypt host +location ~ /.well-known { + allow all; +} diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2 index 10fc4cfda..3ffcbcd9b 100644 --- a/roles/nginx/templates/nginx.conf.j2 +++ b/roles/nginx/templates/nginx.conf.j2 @@ -7,91 +7,90 @@ pid /run/nginx.pid; include /etc/nginx/modules-enabled/*.conf; events { - worker_connections 768; - # multi_accept on; + worker_connections 768; + # multi_accept on; } http { - ## - # Basic Settings - ## + ## + # Basic Settings + ## - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; - # server_tokens off; + # server_tokens off; - server_names_hash_bucket_size 64; - # server_name_in_redirect off; + server_names_hash_bucket_size 64; + # server_name_in_redirect off; - include /etc/nginx/mime.types; - default_type text/html; + include /etc/nginx/mime.types; + default_type text/html; - ## - # SSL Settings - ## + ## + # SSL Settings + ## - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; - ## - # Logging Settings - ## + ## + # Logging Settings + ## - log_format awstats - '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "http_x_forwarded_for"'; + log_format awstats + '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "http_x_forwarded_for"'; - access_log {{ nginx_log_dir }}/access.log awstats; - error_log {{ nginx_log_dir }}/error.log; + access_log {{ nginx_log_dir }}/access.log awstats; + error_log {{ nginx_log_dir }}/error.log; - log_format scripts '$request > $document_root$fastcgi_script_name $fastcgi_path_info'; - access_log {{ nginx_log_dir }}/scripts.log scripts; + log_format scripts '$request > $document_root$fastcgi_script_name $fastcgi_path_info'; + access_log {{ nginx_log_dir }}/scripts.log scripts; + ## + # Gzip Settings + ## - ## - # Gzip Settings - ## + gzip on; + gzip_disable "msie6"; - gzip on; - gzip_disable "msie6"; + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - # gzip_vary on; - # gzip_proxied any; - # gzip_comp_level 6; - # gzip_buffers 16 8k; - # gzip_http_version 1.1; - # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + # 2021-04-17: STANZA BELOW THANKS TO @georgejhunt FOR http://box/jupyterhub + # SEE ALSO IIAB's: roles/jupyterhub/templates/jupyterhub-nginx.conf - # top-level http config for websocket headers - # If Upgrade is defined, Connection = upgrade - # If Upgrade is empty, Connection = close - map $http_upgrade $connection_upgrade { - default upgrade; - '' close; - } + # top-level http config for websocket headers + # If Upgrade is defined, Connection = upgrade + # If Upgrade is empty, Connection = close + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } - ## - # Virtual Host Configs - ## + ## + # Virtual Host Configs + ## + # include a server file which in turn includes conf.d/* + include /etc/nginx/server.conf; - # include a server file which in turn includes conf.d/* - include /etc/nginx/server.conf; - - # include other sites - include /etc/nginx/sites-enabled/*.conf; + # include other sites + include /etc/nginx/sites-enabled/*.conf; - - # define the upstream backend fastcgi for php - upstream php { - server unix:/run/php/php{{ php_version }}-fpm.sock; - } + # define the upstream backend fastcgi for php + upstream php { + server unix:/run/php/php{{ php_version }}-fpm.sock; + } } - From 7454d22302298ea5d2afe0b7ae9184e37de19306 Mon Sep 17 00:00:00 2001 From: A Holt Date: Sat, 17 Apr 2021 08:51:20 -0400 Subject: [PATCH 04/11] roles/nginx/templates/nginx.conf.j2: typo cond.d -> conf.d & doc cleanup --- roles/nginx/templates/nginx.conf.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2 index 3ffcbcd9b..bd23bd626 100644 --- a/roles/nginx/templates/nginx.conf.j2 +++ b/roles/nginx/templates/nginx.conf.j2 @@ -1,5 +1,6 @@ -# IIAB notes: sites-enabled is for server declarations -# cond.d is for location declarations within the main server block +# IIAB notes: +# /etc/nginx/sites-enabled is for server declarations +# /etc/nginx/conf.d is for location declarations within the main server block user www-data; worker_processes auto; From 5f94cc64c76b7a8eb243b2e2a6564e998f45f392 Mon Sep 17 00:00:00 2001 From: A Holt Date: Sat, 17 Apr 2021 10:39:34 -0400 Subject: [PATCH 05/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ae406e37..21238b9ab 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ To learn more about our open community architecture for "offline" education, sta Pre-releases of Internet-in-a-Box (IIAB) undergo continuous QA / continuous integration / continuous deployment and are **strongly recommended!** -Install our latest pre-release using the 1-line installer at: [download.iiab.io](http://download.iiab.io/) +Install our latest pre-release using the 1-line installer at: [**download.iiab.io**](http://download.iiab.io/) You can also consider earlier official releases at: [github.com/iiab/iiab/releases](https://github.com/iiab/iiab/releases) From 6e7ab954977d970939d3d470bd067526cfcdc968 Mon Sep 17 00:00:00 2001 From: A Holt Date: Sat, 17 Apr 2021 15:07:18 -0400 Subject: [PATCH 06/11] roles/nginx/README.md: lowercase role names here --- roles/nginx/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nginx/README.md b/roles/nginx/README.md index 84fa21a1b..79f62cfcb 100644 --- a/roles/nginx/README.md +++ b/roles/nginx/README.md @@ -16,7 +16,7 @@ * Admin Console * captiveportal * IIAB documentation (http://box/info) - * JupyterHub + * jupyterhub * osm-vector-maps * OER2Go/RACHEL modules * usb_lib From 3aaa4b6bd29a0368183451972f38e8db508492d7 Mon Sep 17 00:00:00 2001 From: A Holt Date: Mon, 19 Apr 2021 13:19:59 -0400 Subject: [PATCH 07/11] jupyterhub/tasks/install.yml: Clarify "jupyterhub_venv: /opt/iiab/jupyterhub" --- roles/jupyterhub/tasks/install.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/jupyterhub/tasks/install.yml b/roles/jupyterhub/tasks/install.yml index 8f35b418c..948c5f453 100644 --- a/roles/jupyterhub/tasks/install.yml +++ b/roles/jupyterhub/tasks/install.yml @@ -28,7 +28,7 @@ global: yes state: latest -- name: "Use pip to install into a virtual environment: {{ jupyterhub_venv }}" +- name: "Use pip to install into a virtual environment: {{ jupyterhub_venv }} (~229 MB)" pip: name: - pip @@ -38,7 +38,7 @@ - jupyterlab - jupyterhub_firstuseauthenticator - jupyterhub-systemdspawner - virtualenv: "{{ jupyterhub_venv }}" # /opt/iiab/jupyter + virtualenv: "{{ jupyterhub_venv }}" # /opt/iiab/jupyterhub virtualenv_site_packages: no virtualenv_command: /usr/bin/virtualenv virtualenv_python: python3 From 683e5a558032659dab5dc93b2567c2bf785055ff Mon Sep 17 00:00:00 2001 From: A Holt Date: Mon, 19 Apr 2021 13:26:15 -0400 Subject: [PATCH 08/11] sugarizer/tasks/install.yml: Clarify ~710MB download during git clone --- roles/sugarizer/tasks/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sugarizer/tasks/install.yml b/roles/sugarizer/tasks/install.yml index ac490d70e..b986ca304 100644 --- a/roles/sugarizer/tasks/install.yml +++ b/roles/sugarizer/tasks/install.yml @@ -29,7 +29,7 @@ # 2. DOWNLOAD+LINK /opt/iiab/sugarizer -- name: Clone llaske/sugarizer ({{ sugarizer_git_version }} branch/version) from GitHub to /opt/iiab/{{ sugarizer_dir_version }} (MAY DOWNLOAD 600+ MB) +- name: Clone llaske/sugarizer ({{ sugarizer_git_version }} branch/version) from GitHub to /opt/iiab/{{ sugarizer_dir_version }} (DOWNLOADS ~710 MB) git: repo: https://github.com/llaske/sugarizer dest: "{{ iiab_base }}/{{ sugarizer_dir_version }}" From fdb3e5c84e762fc4c55060feaa14575ede82f6ea Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 21 Apr 2021 18:21:42 -0400 Subject: [PATCH 09/11] nodejs/defaults/main.yml: Clarify Node.js version & version history, per default_vars.yml --- roles/nodejs/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nodejs/defaults/main.yml b/roles/nodejs/defaults/main.yml index 17e7012f9..6e710710d 100644 --- a/roles/nodejs/defaults/main.yml +++ b/roles/nodejs/defaults/main.yml @@ -9,7 +9,7 @@ # Node.js version used by roles/nodejs/tasks/main.yml for 3 roles: # nodered (Node-RED), pbx (Asterix, FreePBX) & sugarizer (Sugarizer) -# nodejs_version: 12.x # was 8.x until 2019-02-02, was 10.x until 2019-12-21 +# nodejs_version: 14.x # was 8.x til 2019-02-02, 10.x til 2019-12-21, 12.x til 2020-10-29 # All above are set in: github.com/iiab/iiab/blob/master/vars/default_vars.yml # If nec, change them by editing /etc/iiab/local_vars.yml prior to installing! From 67d5d1befd91a50b85c0b4e104437fc7113e2fff Mon Sep 17 00:00:00 2001 From: Jerry Vonau Date: Thu, 22 Apr 2021 12:22:38 -0500 Subject: [PATCH 10/11] Update ubuntu-21.yml shipping 13 --- vars/ubuntu-21.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/ubuntu-21.yml b/vars/ubuntu-21.yml index b66db68f7..5f0138fff 100644 --- a/vars/ubuntu-21.yml +++ b/vars/ubuntu-21.yml @@ -24,5 +24,5 @@ sshd_package: openssh-server sshd_service: ssh php_version: 7.4 # 2020-11-14: Change to 8.0 very soon? # "postgresql_version: 11.2" failed (too detailed for /etc/systemd/system/postgresql-iiab.service on Ubuntu 19.04) -postgresql_version: 12 # 2020-11-14: Change to 13 very soon? +postgresql_version: 13 # 2020-11-14: Change to 13 very soon? systemd_location: /lib/systemd/system From 8bd489597873f379323e29c6acb15c94639dc595 Mon Sep 17 00:00:00 2001 From: A Holt Date: Thu, 22 Apr 2021 13:37:14 -0400 Subject: [PATCH 11/11] vars/ubuntu-21.yml: Update PHP & PostgreSQL version# comments (good for 21.04, but 21.10 might complicate things!) --- vars/ubuntu-21.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/ubuntu-21.yml b/vars/ubuntu-21.yml index 5f0138fff..5f7dc7f33 100644 --- a/vars/ubuntu-21.yml +++ b/vars/ubuntu-21.yml @@ -22,7 +22,7 @@ mysql_service: mariadb apache_log: /var/log/apache2/access.log sshd_package: openssh-server sshd_service: ssh -php_version: 7.4 # 2020-11-14: Change to 8.0 very soon? +php_version: 7.4 # 2020-04-22: Will Ubuntu 21.10 require 8.0? # "postgresql_version: 11.2" failed (too detailed for /etc/systemd/system/postgresql-iiab.service on Ubuntu 19.04) -postgresql_version: 13 # 2020-11-14: Change to 13 very soon? +postgresql_version: 13 # 2020-04-22: Will Ubuntu 21.10 require 14? systemd_location: /lib/systemd/system