Merge pull request #2070 from georgejhunt/cap3.1

Cap3.1 on nginx and python 3 [Captive Portal]

roles/2-common/tasks/packages.yml

@@ -33,7 +33,7 @@
     state: present
   when: is_debuntu | bool
 
-- name: "Install 23 common packages: acpid, bridge-utils, bzip2, curl, gawk, hostapd, htop, i2c-tools, logrotate, make, mlocate, netmask, net-tools, ntfs-3g, pandoc, pastebinit, rsync, sudo, tar, unzip, usbmount, usbutils, wget"
+- name: "Install 24 common packages: acpid, bridge-utils, bzip2, curl, gawk, hostapd, htop, i2c-tools, logrotate, make, mlocate, netmask, net-tools, ntfs-3g, pandoc, pastebinit, rsync, sqlite3,sudo, tar, unzip, usbmount, usbutils, wget"
   package:
     name:
       - acpid
@@ -56,6 +56,7 @@
       - pandoc
       - pastebinit
       - rsync
+      - sqlite3
       - sudo
       - tar
       - unzip

roles/9-local-addons/tasks/main.yml

@@ -18,9 +18,9 @@
 
 # To be ported soon
 - name: CAPTIVE PORTAL
-  include_tasks: roles/captive-portal/tasks/main.yml
+  include_tasks: roles/captiveportal/tasks/main.yml
-  when: captive_portal_install | bool
+  when: captiveportal_install | bool
-  tags: base, captive-portal, network, domain
+  tags: base, captiveportal, network, domain
 
 - name: MINETEST
   include_role:

roles/captive-portal/tasks/main.yml

@@ -1,153 +0,0 @@
-- name: Download & install python-dateutil, sqlite3
-  package:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-dateutil
-    - sqlite3    # @georgejhunt hopes to move this to 2-common (or more likely 3-base-server, alongside MySQL) in October 2018
-
-- name: Install libapache2-mod-wsgi (debuntu)
-  package:
-    name: libapache2-mod-wsgi
-    state: present
-  when: is_debuntu | bool
-
-- name: Install mod_wsgi (not debuntu)
-  package:
-    name: mod_wsgi
-    state: present
-  when: not is_debuntu
-
-- name: Create directory /opt/iiab/captive-portal for scripts & templates
-  file:
-    path: /opt/iiab/captive-portal
-    state: directory
-    owner: "{{ apache_user }}"
-
-- name: 'Copy scripts: checkurls, capture-wsgi.py'
-  template:
-    src: "{{ item.src }}"
-    dest: /opt/iiab/captive-portal/
-    mode: "{{ item.mode }}"
-  with_items:
-    - { src: roles/captive-portal/templates/checkurls, mode: '0644' }
-    - { src: roles/captive-portal/templates/capture-wsgi.py, mode: '0755' }
-
-- name: 'Copy templates: simple.template, mac.template'
-  copy:
-    src: "{{ item }}"
-    dest: /opt/iiab/captive-portal/
-  with_items:
-    - roles/captive-portal/files/simple.template
-    - roles/captive-portal/files/mac.template
-
-- name: Copy iiab-catch & iiab-uncatch into /usr/bin/
-  template:
-    src: "{{ item }}"
-    dest: /usr/bin/
-    owner: root
-    group: root
-    mode: 0755
-  with_items:
-    - roles/captive-portal/templates/iiab-catch
-    - roles/captive-portal/templates/iiab-uncatch
-
-- name: Run iiab-uncatch to generate diversion lists for dnsmasq and apache2
-  shell: /usr/bin/iiab-uncatch
-     
-#- name: Install systemd unit file captive-portal.service from template
-#  template:
-#    src: roles/captive-portal/templates/captive-portal.service.j2
-#    dest: /etc/systemd/system/captive-portal.service
-#    owner: root
-#    group: root
-#    mode: 0644
-
-- name: Install Apache's captive-portal.conf from template if captive_portal_enabled
-  template:
-    src: roles/captive-portal/templates/001-captive-portal.conf
-    dest: /etc/{{ apache_config_dir }}/001-captive-portal.conf
-    owner: root
-    group: root
-    mode: 0644
-  when: captive_portal_enabled | bool
-
-- name: Enable Apache's captive-portal.conf if captive_portal_enabled (debuntu)
-  file:
-    src: /etc/apache2/sites-available/001-captive-portal.conf
-    path: /etc/apache2/sites-enabled/001-captive-portal.conf
-    state: link
-  when: captive_portal_enabled and is_debuntu
-
-- name: Enable Apache's default-ssl.conf if captive_portal_enabled (debuntu)
-  file:
-    src: /etc/apache2/sites-available/default-ssl.conf
-    path: /etc/apache2/sites-enabled/default-ssl.conf
-    state: link
-  when: captive_portal_enabled and is_debuntu
-
-#- name: Enable & Start systemd service captive-portal.service if captive_portal_enabled
-#  systemd:
-#    name: captive-portal.service
-#    daemon-reload: yes
-#    enabled: yes
-#    state: started
-#  when: captive_portal_enabled | bool
-
-#- name: Disable & Stop captive-portal.service if not captive_portal_enabled
-#  systemd:
-#    name: captive-portal.service
-#    enabled: no
-#    state: stopped
-#  when: not captive_portal_enabled
-
-- name: Disable Apache's captive-portal.conf if not captive_portal_enabled (debuntu)
-  file:
-    path: /etc/apache2/sites-enabled/001-captive-portal.conf
-    state: absent
-  when: not captive_portal_enabled and is_debuntu
-
-- name: Disable Apache's default-ssl.conf if not captive_portal_enabled (debuntu)
-  file:
-    path: /etc/apache2/sites-enabled/default-ssl.conf
-    state: absent
-  when: not captive_portal_enabled and is_debuntu
-
-- name: Make sure dnsmasq is not diverting if not captive_portal_enabled
-  file:
-    path: /etc/dnsmasq.d/capture
-    state: absent
-  when: not captive_portal_enabled
-
-- name: Add 'captive_portal_installed' variable values to {{ iiab_state_file }}
-  lineinfile:
-    dest: "{{ iiab_state_file }}"
-    regexp: '^captive_portal_installed'
-    line: 'captive_portal_installed: True'
-    state: present
-
-- name: Restart Apache service ({{ apache_service }})    # i.e. apache2 on most distros
-  systemd: 
-    name: "{{ apache_service }}"
-    state: restarted
-
-#- name: Restart dnsmasq
-#  systemd:
-#    name: dnsmasq
-#    state: restarted
-#  when: dnsmasq_enabled | bool
-
-# ABOVE DOES NOT WORK ON UBUNTU 16.04 -- what follows is a crude hack (seems to work!)
-
-- name: Stop dnsmasq
-  systemd:
-    name: dnsmasq
-    state: stopped
-  when: dnsmasq_enabled | bool
-
-- name: Start dnsmasq
-  systemd:
-    name: dnsmasq
-    state: started
-  when: dnsmasq_enabled | bool
-  

roles/captive-portal/templates/001-captive-portal.conf

@@ -1,43 +0,0 @@
-<VirtualHost _default_:80>
-   ErrorLog /var/log/apache2/error.log
-   CustomLog /var/log/apache2/access.log combined
-   <Directory {{ doc_root }}>
-       Options Indexes FollowSymLinks
-       AllowOverride None
-       Require all granted
-   </Directory>
-</VirtualHost>
-
-<VirtualHost *:80>
-	# The ServerName directive sets the request scheme, hostname and port that
-	# the server uses to identify itself. This is used when creating
-	# redirection URLs. In the context of virtual hosts, the ServerName
-	# specifies what hostname must appear in the request's Host: header to
-	# match this virtual host. For the default virtual host (this file) this
-	# value is not decisive as it is used as a last resort host regardless.
-	# However, you must set it for any further virtual host explicitly.
-	ServerName iiab.io
-   Include /etc/apache2/capture
-#   ProxyPreserveHost On
-#   ProxyPass / http://box.lan:{{ captive_portal_port }}/
-#   ProxyPassReverse / http://box.lan:{{ captive_portal_port }}/
-   ErrorLog /var/log/apache2/cp_error.log
-WSGIScriptAlias / /opt/iiab/captive-portal/capture-wsgi.py
-#WSGIScriptAlias / /opt/iiab/captive-portal/test.py
-WSGIScriptReloading On
-   <Directory /opt/iiab/captive-portal>
-       AllowOverride None
-       Require all granted
-   </Directory>
-
-</VirtualHost>
-
-<VirtualHost 127.0.0.1:80>
-   ErrorLog /var/log/apache2/error.log
-   CustomLog /var/log/apache2/access.log combined
-   <Directory /library/www/html>
-       Options Indexes FollowSymLinks
-       AllowOverride None
-       Require all granted
-   </Directory>
-</VirtualHost>

roles/captive-portal/templates/captive-portal.service.j2.deprecated

@@ -1,15 +0,0 @@
-[Unit]
-Description=Captive portal
-After=syslog.target
-
-[Service]
-Type=simple
-User=root
-Group=root
-WorkingDirectory=/opt/iiab/captive-portal
-ExecStart=/opt/iiab/captive-portal/capture-wsgi.py -l
-StandardOutput=syslog
-StandardError=syslog
-
-[Install]
-WantedBy=multi-user.target

roles/captive-portal/templates/iiab-catch

@@ -1,9 +0,0 @@
-#!/bin/bash -x
-# substitute our own server to catch OS connectivity checking URL's
-
-systemctl stop {{ apache_service }}
-# systemctl stop captive-portal
-echo address=/#/172.18.96.1 > /etc/dnsmasq.d/capture
-/opt/iiab/captive-portal/capture-wsgi.py -d &
-# write the pid just started
-echo $! > /opt/iiab/captive-portal/pid

roles/captive-portal/templates/iiab-uncatch

@@ -1,15 +0,0 @@
-#!/bin/bash -x
-# Turn off URL recording mode, and return to serving with apache2
-
-kill $(cat /opt/iiab/captive-portal/pid)
-# during testing, I start capture by hand -- recorded pid may be stale
-pid=$(ps aux | grep "capture-wsgi.py -d" | grep -v grep | awk '{print $2}')
-if [ -n "$pid" ]; then
-    kill $pid
-fi
-awk '{print("address=/" $1 "/172.18.96.1")}' /opt/iiab/captive-portal/checkurls > /etc/dnsmasq.d/capture
-echo "#following tells windows 7 that captive portal is active" >>/etc/dnsmasq.d/capture
-echo "address=/dns.msftncsi.com/131.107.255.255" >> /etc/dnsmasq.d/capture
-awk '{print("ServerAlias ",$1)}' /opt/iiab/captive-portal/checkurls > /etc/apache2/capture
-# systemctl start captive-portal
-systemctl start {{ apache_service }}

roles/captiveportal/README.md

@@ -0,0 +1,22 @@
+## Theory of Operation
+
+* The captive portal function is a feature of most modern operating systems. With the increased use of https/ssl (secure sockets layer), the automatic diversion to a specific web page runs the risk of being detected as a "man in the middle" attack.
+* Each Operating System (OS) provides a mechanism that IIAB can use to break into a conversation, when SSL is not being used. This is an initial attempt by the OS to talk to one of its own web sites, to determine if the host os is connected to the internet. It is always performed without SSL.
+* The IIAB captive portal uses a list of these OS supported web sites, and diverts these requests to the IIAB server, which in turn forwards to the IIAB home page.
+
+## Components of the IIAB Captive Portal
+
+* Files used
+    1. checkurls -- the list of urls use by at least one of the OS's.
+    1. iiab-divert-to-nginx -- Bash script writes dnsmasq config file which points to IIAB server
+    1. iiab-make-cp-servers.py -- Python script writes nginx configuration file to /etc/nginx/sites-enabled
+    1. capture-wsgi.py -- the script which determines the client agent, records it in sqlite database, and responds with redirects as appropriate for each OS.
+    1. uwsgi-captiveportal.service -- systemd unit file which runs uwsgi which makes capture-wsgi.py available on port 9090.
+    
+ ## Extending and Debugging Captive Portal
+ * The python capture script can be run interactively in terminal (use systemctl stop uwsgi-captiveportal to free up the port). This will expose any python errors easily.
+ * Run the capture-wsgi.py with "-l" in a terminal to increase logging to /var/log/apache2/portal.log
+ * To discover untrapped urls, "apt-get install tcpdump", and "tcpdump -i br0 capture.tcp". I transfer this file to a machine with a GUI, and wireshark to interpret the conversations on the wire. The DNS packets are the ones to look for.
+ 
+ ## Known Problems
+ 1. On Android 5-7, the browser which is brought up, during the association process, is a 'walled garden' and I cannot find a way out. This browser is not very modern, and continuously displays the "sign in to Wi-Fi network" button -- with an annoying beep.

roles/captiveportal/defaults/main.yml

@@ -1,7 +1,7 @@
 # captive_portal_install: False
 # captive_portal_enabled: False
 
-# captive_portal_port: 9090
+# captiveportal_port: 9090
 
 # All above are set in: github.com/iiab/iiab/blob/master/vars/default_vars.yml
 # If nec, change them by editing /etc/iiab/local_vars.yml prior to installing!

roles/captiveportal/files/mac.template

@@ -8,16 +8,17 @@
           #header {
              display: block;
              height: 120px;
-             width:1024px;
+             width:900px;
              background: #000 url('iiab_banner6.png') no-repeat 0 0;
              border-radius: 5px;
              margin: 5px;
+             object-fit: cover;
           }
          body {
              background-color: #CBFFAA;
              font-family: sans-serif;
              font-size: 100%;
-             width: 1024px;
+             width: 900px;
              margin: 3px;
          }
       }

roles/captiveportal/files/simple.template

@@ -69,8 +69,14 @@
     <script>
      var w = window.innerWidth;
      function homeclick(){
+        $.ajax({
+         url: "/home_selected",
+         async: false,
+         success: function( data ){
+           console.log(data);
+         }
+        });
         window.open("http://{{ FQDN }}","_system");
-        $.ajax("/home_selected");
      }
 
     </script>
@@ -93,7 +99,6 @@
       <br><br>
       <br><br>
       <br><br>
-      <!--     <a class="button" href="http://box.lan/home">{{ btn1 }}</a> -->
          <a class="button" onclick="homeclick()">{{ btn1 }}</a>
       </div>
   </body>

roles/captiveportal/tasks/main.yml

@@ -0,0 +1,117 @@
+- name: Download & install python-dateutil, sqlite3
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python3-dateutil
+    - python3-jinja2
+
+- name: Create directory /opt/iiab/captiveportal for scripts & templates
+  file:
+    path: /opt/iiab/captiveportal
+    state: directory
+    owner: "{{ apache_user }}"
+
+- name: 'Copy scripts: checkurls'
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: "{{ item.mode }}"
+  with_items:
+    - { src: roles/captiveportal/templates/checkurls, mode: '0644', dest: /opt/iiab/captiveportal/ }
+    - { src: roles/captiveportal/templates/iiab-make-cp-servers.py, mode: '0755', dest: /usr/sbin/ }
+    - { src: roles/captiveportal/templates/iiab-divert-to-nginx, mode: '0755', dest: /usr/sbin/ }
+
+- name: Put put the python script that creates the server in place
+  template:
+    src: roles/captiveportal/templates/capture-wsgi.py 
+    mode: '0755' 
+    dest: /opt/iiab/captiveportal/ 
+
+- name: 'Copy templates: simple.template, mac.template'
+  copy:
+    src: "{{ item }}"
+    dest: /opt/iiab/captiveportal/
+  with_items:
+    - roles/captiveportal/files/simple.template
+    - roles/captiveportal/files/mac.template
+
+- name: Copy uWSGI config file
+  template:
+    src: roles/captiveportal/templates/captiveportal.ini.j2
+    dest: /opt/iiab/captiveportal/captiveportal.ini
+
+- name: Copy unit file for uWSGI service
+  template:
+    src: roles/captiveportal/templates/uwsgi-captiveportal.service
+    dest: /etc/systemd/system/
+
+- name: Start or restart server which responds to browsers trying to detect a captive portal
+  systemd: 
+    name: uwsgi-captiveportal.service
+    state: restarted
+    enabled: True
+  when: captiveportal_enabled | bool
+
+- name: Stop uWSGI server if captive portal has been disabled
+  systemd: 
+    name: uwsgi-captiveportal.service
+    state: stopped
+    enabled: False
+  when: not captiveportal_enabled | bool
+
+- name: Run divert to generate diversion lists for nginx
+  shell: /usr/sbin/iiab-divert-to-nginx
+     
+- name: Run script to generate nginx servers from checkurls input list
+  command: /usr/sbin/iiab-make-cp-servers.py
+  args:
+      creates: /etc/nginx/sites-available/capture.conf
+  
+- name: Enable nginx to service the sites in checkurls list
+  file:
+    src: /etc/nginx/sites-available/capture.conf
+    path: /etc/nginx/sites-enabled/capture.conf
+    state: link
+  when: captiveportal_enabled | bool
+
+- name: Disable nginx to location definitions for checkurls
+  file:
+    src: /etc/nginx/sites-available/capture.conf
+    path: /etc/nginx/sites-enabled/capture.conf
+    state: absent
+  when: not captiveportal_enabled | bool
+
+- name: Make sure dnsmasq is not diverting if not captiveportal_enabled
+  file:
+    path: /etc/dnsmasq.d/capture
+    state: absent
+  when: not captiveportal_enabled
+
+- name: Add 'captiveportal_installed' variable values to {{ iiab_state_file }}
+  lineinfile:
+    dest: "{{ iiab_state_file }}"
+    regexp: '^captiveportal_installed'
+    line: 'captiveportal_installed: True'
+    state: present
+
+#- name: Restart dnsmasq
+#  systemd:
+#    name: dnsmasq
+#    state: restarted
+#  when: dnsmasq_enabled | bool
+
+# ABOVE DOES NOT WORK ON UBUNTU 16.04 -- what follows is a crude hack (seems to work!)
+
+- name: Stop dnsmasq
+  systemd:
+    name: dnsmasq
+    state: stopped
+  when: dnsmasq_enabled | bool
+
+- name: Start dnsmasq
+  systemd:
+    name: dnsmasq
+    state: started
+  when: dnsmasq_enabled | bool
+  

roles/captiveportal/templates/captiveportal.ini.j2

@@ -0,0 +1,9 @@
+[uwsgi]
+   uid = {{ apache_user }}
+   gid = {{ apache_user }}
+   http-socket = :{{ captiveportal_port }}
+   chdir = /opt/iiab/captiveportal
+   wsgi-file = capture-wsgi.py
+   master = true
+   plugins = python3
+   py-autoreload = 2

roles/captiveportal/templates/capture-wsgi.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 # -*- coding: utf-8 -*-
 # using Python's bundled WSGI server
 
@@ -13,6 +13,7 @@ import sys
 from jinja2 import Environment, FileSystemLoader
 import sqlite3
 import re
+from iiab.iiab_lib import get_iiab_env
 
 # Notes on timeout strategy
 # every client timestamp is recorded into current_ts
@@ -23,7 +24,7 @@ import re
 # 
 
 # Create the jinja2 environment.
-CAPTIVE_PORTAL_BASE = "/opt/iiab/captive-portal"
+CAPTIVE_PORTAL_BASE = "/opt/iiab/captiveportal"
 j2_env = Environment(loader=FileSystemLoader(CAPTIVE_PORTAL_BASE),trim_blocks=True)
 
 # Define time outs
@@ -34,61 +35,33 @@ PORTAL_TO = 20 # delay after triggered by ajax upon click of link to home page
 
 
 # Get the IIAB variables
-sys.path.append('/etc/iiab/')
-from iiab_env import get_iiab_env
 doc_root = get_iiab_env("WWWROOT")
 fully_qualified_domain_name = get_iiab_env("FQDN")
 
 
+loggingLevel = "ERROR"
+#loggingLevel = "DEBUG"
+if len(sys.argv) > 1:
+   if sys.argv[1] == '-l':
+      loggingLevel = "DEBUG"
+      
 # set up some logging -- selectable for diagnostics
-# Create dummy iostream to capture stderr and stdout
-class StreamToLogger(object):
-    """
-    Fake file-like stream object that redirects writes to a logger instance.
-    """
-    def __init__(self, logger, log_level=logging.INFO):
-        self.logger = logger
-        self.log_level = log_level
-        self.linebuf = ''
-
-    def write(self, buf):
-        for line in buf.rstrip().splitlines():
-            self.logger.log(self.log_level, line.rstrip())
-
-#if len(sys.argv) > 1 and sys.argv[1] == '-l':
-if True:
-    loggingLevel = logging.DEBUG
-    try:
-      os.remove('/var/log/apache2/portal.log')
-    except:
-      pass
-else:
-    loggingLevel = logging.ERROR
-
-# divert stdout and stderr to logger
 logging.basicConfig(filename='/var/log/apache2/portal.log',format='%(asctime)s.%(msecs)03d:%(name)s:%(message)s', datefmt='%M:%S',level=loggingLevel)
 logger = logging.getLogger('/var/log/apache2/portal.log')
 handler = RotatingFileHandler("/var/log/apache2/portal.log", maxBytes=100000, backupCount=2)
 logger.addHandler(handler)
 
-stdout_logger = logging.getLogger('STDOUT')
+PORT={{ captiveportal_port }}
-sl = StreamToLogger(stdout_logger, logging.ERROR)
+#PORT=9090
-sys.stdout = sl
-
-stderr_logger = logging.getLogger('STDERR')
-sl = StreamToLogger(stderr_logger, logging.ERROR)
-sys.stderr = sl
-PORT={{ captive_portal_port }}
 
 
 # Define globals
-ANDROID_TRIGGERED=False
 
 logger.debug("")
 logger.debug('##########################################')
 # what language are we speaking?
 lang = os.environ['LANG'][0:2]
-logger.debug('speaking: %s'%lang)
+logger.debug('speaking: {}'.format(lang))
 
 def tstamp(dtime):
     '''return a UNIX style seconds since 1970 for datetime input'''
@@ -141,8 +114,7 @@ def timeout_info(ip):
 def is_inactive(ip):
     ts=tstamp(datetime.datetime.now(tzutc()))
     current_ts, last_ts, send204after = timeout_info(ip) 
-    logger.debug("In is_inactive. current_ts:%s. last_ts:%s. send204after:%s"%\
+    logger.debug("In is_inactive. current_ts:{}. last_ts:{}. send204after:{}".format(current_ts,last_ts,send204after,))
-            (current_ts,last_ts,send204after,))
     if not last_ts:
         return True
     if ts - int(last_ts) > INACTIVITY_TO:
@@ -154,7 +126,7 @@ def is_after204_timeout(ip):
     ts=tstamp(datetime.datetime.now(tzutc()))
     current_ts, last_ts, send204after = timeout_info(ip) 
     if send204after == 0: return False
-    logger.debug("function: is_after204_timeout send204after:%s current: %s"%(send204after,ts,))
+    logger.debug("function: is_after204_timeout send204after:{} current: {}".format(send204after,ts,))
     if not send204after:
         return False
     if ts - int(send204after) > 0:
@@ -163,12 +135,10 @@ def is_after204_timeout(ip):
         return False
 
 def set_204after(ip,value):
-    global ANDROID_TRIGGERED
     ts=tstamp(datetime.datetime.now(tzutc()))
     sql = 'UPDATE users SET send204after = ?  where ip = ?'
     c.execute(sql,(ts + value,ip,))
     conn.commit()
-    ANDROID_TRIGGERED = False
 
 def set_lasttimestamp(ip):
     ts=tstamp(datetime.datetime.now(tzutc()))
@@ -178,31 +148,33 @@ def set_lasttimestamp(ip):
 
 #  ###################  Action routines based on OS  ################3
 def microsoft(environ,start_response):
+    logger.debug('in microsoft')
     # firefox -- seems both mac and Windows use it
     agent = environ.get('HTTP_USER_AGENT','default_agent')
     if agent.startswith('Mozilla'):
+       logger.debug("sending microsoft redirect for agent Mozilla")
        return home(environ, start_response) 
-    logger.debug("sending microsoft redirect")
+    response_body = b""
-    response_body = ""
     status = '302 Moved Temporarily'
-    response_headers = [('Location','http://box.lan/home'),
+    response_headers = [('Location','http://' + fully_qualified_domain_name + '{{ captiveportal_splash_page }}'),
             ('Content-type','text/html'),
             ('Content-Length',str(len(response_body)))]
     start_response(status, response_headers)
+    logger.debug("redirect to home. Status: %s Headers: %s"%(status,repr(response_headers)))
     return [response_body]
 
 def home(environ,start_response):
     logger.debug("sending direct to home")
-    response_body = ""
+    response_body = b""
     status = '302 Moved Temporarily'
-    response_headers = [('Location','http://' + fully_qualified_domain_name + '/home'),
+    response_headers = [('Location','http://' + fully_qualified_domain_name + '{{ captiveportal_splash_page }}'),
             ('Content-type','text/html'),
             ('Content-Length',str(len(response_body)))]
     start_response(status, response_headers)
+    logger.debug("redirect to home. Status: %s Headers: %s"%(status,repr(response_headers)))
     return [response_body]
 
 def android(environ, start_response):
-    global ANDROID_TRIGGERED
     if  environ.get('HTTP_X_FORWARDED_FOR'):
         ip = environ['HTTP_X_FORWARDED_FOR'].strip()
     else: 
@@ -211,16 +183,16 @@ def android(environ, start_response):
     if system_version is None:
        return  put_302(environ, start_response)
     if system_version[0:1] < '6':
-        logger.debug("system < 6:%s"%system_version)
+        logger.debug("system < 6:{}".format(system_version))
         location = '/android_splash'
         set_204after(ip,0)
     elif system_version[:1] >= '7':
-        location = "http://" + fully_qualified_domain_name + "/home"
+        location = "http://" + fully_qualified_domain_name + '{{ captiveportal_splash_page }}'
     else:
         #set_204after(ip,20)
         location = '/android_https'
     agent = environ.get('HTTP_USER_AGENT','default_agent')
-    response_body = "hello"
+    response_body = b"hello"
     status = '302 Moved Temporarily'
     response_headers = [('Location',location)]
     start_response(status, response_headers)
@@ -229,10 +201,10 @@ def android(environ, start_response):
 def android_splash(environ, start_response):
     en_txt={ 'message':"Click on the button to go to the IIAB home page",\
             'btn1':"GO TO IIAB HOME PAGE", \
-            "FQDN": fully_qualified_domain_name, \
+            "FQDN": fully_qualified_domain_name + '{{ captiveportal_splash_page }}', \
             'doc_root':get_iiab_env("WWWROOT") }
     es_txt={ 'message':"Haga clic en el botón para ir a la página de inicio de IIAB",\
-            "FQDN": fully_qualified_domain_name, \
+            "FQDN": fully_qualified_domain_name + '{{ captiveportal_splash_page }}', \
             'btn1':"IIAB",'doc_root':get_iiab_env("WWWROOT")}
     txt = en_txt
     if lang == "en":
@@ -240,6 +212,7 @@ def android_splash(environ, start_response):
     elif lang == "es":
         txt = es_txt
     response_body = str(j2_env.get_template("simple.template").render(**txt))
+    response_body = response_body.encode()
     status = '200 OK'
     response_headers = [('Content-type','text/html'),
             ('Content-Length',str(len(response_body)))]
@@ -250,10 +223,10 @@ def android_https(environ, start_response):
     en_txt={ 'message':"""Please ignore the SECURITY warning which appears after clicking the first button""",\
              'btn2':'Click this first Go to the browser we need',\
              'btn1':'Then click this to go to IIAB home page',\
-             "FQDN": fully_qualified_domain_name, \
+             "FQDN": fully_qualified_domain_name + '{{ captiveportal_splash_page }}', \
             'doc_root':get_iiab_env("WWWROOT") }
     es_txt={ 'message':"Haga clic en el botón para ir a la página de inicio de IIAB",\
-            "FQDN": fully_qualified_domain_name, \
+            "FQDN": fully_qualified_domain_name + '{{ captiveportal_splash_page }}', \
             'btn1':"IIAB",'doc_root':get_iiab_env("WWWROOT")}
     txt = en_txt
     if lang == "en":
@@ -261,6 +234,7 @@ def android_https(environ, start_response):
     elif lang == "es":
         txt = es_txt
     response_body = str(j2_env.get_template("simple.template").render(**txt))
+    response_body = response_body.encode()
     status = '200 OK'
     response_headers = [('Content-type','text/html'),
             ('Content-Length',str(len(response_body)))]
@@ -268,13 +242,14 @@ def android_https(environ, start_response):
     return [response_body]
 
 def mac_splash(environ,start_response):
+    logger.debug('in mac_splash')
     logger.debug("in function mac_splash")
-    en_txt={ 'message':"Click on the button to go to the IIAB home page",\
+    en_txt={ 'message': "Click on the button to go to the IIAB home page",\
-            'btn1':"GO TO IIAB HOME PAGE",'success_token': 'Success',
+            'btn1': "GO TO IIAB HOME PAGE",'success_token': 'Success',
-            "FQDN": fully_qualified_domain_name, \
+            "FQDN": fully_qualified_domain_name + '{{ captiveportal_splash_page }}', \
             'doc_root':get_iiab_env("WWWROOT")}
     es_txt={ 'message':"Haga clic en el botón para ir a la página de inicio de IIAB",\
-            "FQDN": fully_qualified_domain_name, \
+            "FQDN": fully_qualified_domain_name + '{{ captiveportal_splash_page }}', \
             'btn1':"IIAB",'doc_root':get_iiab_env("WWWROOT")}
     txt = en_txt
     if lang == "en":
@@ -283,6 +258,7 @@ def mac_splash(environ,start_response):
         txt = es_txt
     set_lasttimestamp(ip)
     response_body = str(j2_env.get_template("mac.template").render(**txt))
+    response_body = response_body.encode()
     status = '200 Success'
     response_headers = [('Content-type','text/html'),
             ('Content-Length',str(len(response_body)))]
@@ -290,6 +266,7 @@ def mac_splash(environ,start_response):
     return [response_body]
 
 def macintosh(environ, start_response):
+    logger.debug('in macintosh')
     global ip
     logger.debug("in function mcintosh")
     #print >> sys.stderr , "Geo Print to stderr" + environ['HTTP_HOST']
@@ -302,6 +279,7 @@ def macintosh(environ, start_response):
         response_body = """<html><head><script>
             window.location.reload(true)
             </script></body></html>"""
+        response_body = response_body.encode()
         status = '302 Moved Temporarily'
         response_headers = [('content','text/html')]
         start_response(status, response_headers)
@@ -309,18 +287,12 @@ def macintosh(environ, start_response):
     else:
         return mac_splash(environ,start_response)
 
-def microsoft_connect(environ,start_response):
-    status = '200 ok'
-    headers = [('Content-type', 'text/html')]
-    start_response(status, headers)
-    return ["Microsoft Connect Test"]
-
 # =============  Return html pages  ============================
 def banner(environ, start_response):
     status = '200 OK'
     headers = [('Content-type', 'image/png')]
     start_response(status, headers)
-    image = open("%s/js-menu/menu-files/images/iiab_banner6.png"%doc_root, "rb").read()
+    image = open("{}/js-menu/menu-files/images/iiab_banner6.png".format(doc_root), "rb").read()
     return [image]
 
 def bootstrap(environ, start_response):
@@ -328,7 +300,7 @@ def bootstrap(environ, start_response):
     status = '200 OK'
     headers = [('Content-type', 'text/javascript')]
     start_response(status, headers)
-    boot = open("%s/common/js/bootstrap.min.js"%doc_root, "rb").read() 
+    boot = open("{}/common/js/bootstrap.min.js".format(doc_root), "rb").read() 
     return [boot]
 
 def jquery(environ, start_response):
@@ -336,7 +308,7 @@ def jquery(environ, start_response):
     status = '200 OK'
     headers = [('Content-type', 'text/javascript')]
     start_response(status, headers)
-    boot = open("%s/common/js/jquery.min.js"%doc_root, "rb").read() 
+    boot = open("{}/common/js/jquery.min.js".format(doc_root), "rb").read() 
     return [boot]
 
 def bootstrap_css(environ, start_response):
@@ -344,25 +316,25 @@ def bootstrap_css(environ, start_response):
     status = '200 OK'
     headers = [('Content-type', 'text/css')]
     start_response(status, headers)
-    boot = open("%s/common/css/bootstrap.min.css"%doc_root, "rb").read() 
+    boot = open("{}/common/css/bootstrap.min.css".format(doc_root), "rb").read() 
     return [boot]
 
 def null(environ, start_response):
     status = '404 Not Found'
     headers = [('Content-type', 'text/html')]
     start_response(status, headers)
-    return [""]
+    return [b""]
 
 def success(environ, start_response):
     status = '200 ok'
-    html = '<html><head><title>Success</title></head><body>Success</body></html>'
+    html = b'<html><head><title>Success</title></head><body>Success</body></html>'
     headers = [('Content-type', 'text/html')]
     start_response(status, headers)
     return [html]
 
 def put_204(environ, start_response):
     status = '204 No Data'
-    response_body = ''
+    response_body = b''
     response_headers = [('Content-type','text/html'),
             ('Content-Length',str(len(response_body)))]
     start_response(status, response_headers)
@@ -371,8 +343,8 @@ def put_204(environ, start_response):
 
 def put_302(environ, start_response):
     status = '302 Moved Temporarily'
-    response_body = ''
+    response_body = b''
-    location = "http://" + fully_qualified_domain_name + "/home"
+    location = "http://" + fully_qualified_domain_name + '{{ captiveportal_splash_page }}'
     response_headers = [('Content-type','text/html'),
             ('Location',location), 
             ('Content-Length',str(len(response_body)))]
@@ -412,23 +384,22 @@ def application (environ, start_response):
    global CATCH
    global LIST
    global INACTIVITY_TO
-   global ANDROID_TRIGGERED
 
    if  'HTTP_X_FORWARDED_FOR' in environ:
       ip = environ['HTTP_X_FORWARDED_FOR'].strip()
    else:
-      data = ['%s: %s\n' % (key, value) for key, value in sorted(environ.items()) ]
+      data = ['{}: {}\n'.format(key, value) for key, value in sorted(environ.items()) ]
-      #logger.debug("need the correct ip:%s"%data)
+      #logger.debug("need the correct ip:{}".format(data))
       ip = environ['REMOTE_ADDR'].strip()
-   cmd="arp -an %s|gawk \'{print $4}\'" % ip
+   cmd="arp -an %s|gawk \'{print $4}\'"%(ip)
    mac = subprocess.check_output(cmd, shell=True)
    data = []
-   data.append("host: %s\n"%environ['HTTP_HOST'])
+   data.append("host: {}\n".format(environ['HTTP_HOST']))
-   data.append("path: %s\n"%environ['PATH_INFO'])
+   data.append("path: {}\n".format(environ['PATH_INFO']))
-   data.append("query: %s\n"%environ['QUERY_STRING'])
+   data.append("query: {}\n".format(environ['QUERY_STRING']))
-   data.append("ip: %s\n"%ip)
+   data.append("ip: {}\n".format(ip))
    agent = environ.get('HTTP_USER_AGENT','default_agent')
-   data.append("AGENT: %s\n"%agent)
+   data.append("AGENT: {}\n".format(agent))
    logger.debug(data)
    #print(data)
    found = False
@@ -441,7 +412,7 @@ def application (environ, start_response):
    sql = "UPDATE users SET current_ts = ? where ip = ?" 
    c.execute(sql,(ts,ip,))
    if c.rowcount == 0:
-      logger.debug("failed UPDATE  users SET current_ts = %s WHERE ip = %s"%(ts,ip,)) 
+      logger.debug("failed UPDATE  users SET current_ts = {} WHERE ip = {}".format(ts,ip,)) 
    conn.commit()
    ymd=datetime.datetime.today().strftime("%y%m%d-%H%M")
 
@@ -469,17 +440,18 @@ def application (environ, start_response):
    if  environ['PATH_INFO'] == "/home_selected":
       # the js link to home page triggers this ajax url 
       # mark the sign-in conversation completed, return 204 or Success or Success
-      ANDROID_TRIGGERED = True
+      #data = ['{}: {}\n'.format(key, value) for key, value in sorted(environ.items()) ]
-      #data = ['%s: %s\n' % (key, value) for key, value in sorted(environ.items()) ]
+      #logger.debug("need the correct ip:{}".format(data))
-      #logger.debug("need the correct ip:%s"%data)
       logger.debug("function: home_selected. Setting flag to return_204")
       #print("setting flag to return_204")
       set_204after(ip,PORTAL_TO)
       set_lasttimestamp(ip)
       status = '200 OK'
-      headers = [('Content-type', 'text/html')]
+      response_body = b''
-      start_response(status, headers)
+      response_headers = [('Content-type','text/html'),
-      return [""]
+            ('Content-Length',str(len(response_body)))]
+      start_response(status, response_headers)
+      return [response_body]
 
    #### parse OS platform based upon URL  ##################
    # mac
@@ -514,7 +486,7 @@ def application (environ, start_response):
       environ['PATH_INFO'] == "/gen_204" or\
       environ['HTTP_HOST'] == "connectivitycheck.gstatic.com":
       current_ts, last_ts, send204after = timeout_info(ip) 
-      logger.debug("current_ts: %s last_ts: %s send204after: %s"%(current_ts, last_ts, send204after,))
+      logger.debug("current_ts: {} last_ts: {} send204after: {}".format(current_ts, last_ts, send204after,))
       if not last_ts or (ts - int(last_ts) > INACTIVITY_TO):
           return android(environ, start_response) 
       elif is_after204_timeout(ip):
@@ -533,7 +505,7 @@ def application (environ, start_response):
      environ['HTTP_HOST'] == "teredo.ipv6.microsoft.com.nsatc.net": 
      return microsoft(environ, start_response) 
 
-   logger.debug("executing the default 302 response. [%s"%data)
+   logger.debug("executing the default 302 response. [{}".format(data))
    return put_302(environ,start_response)
 
 # Instantiate the server
@@ -545,5 +517,5 @@ if __name__ == "__main__":
     )
 
     httpd.serve_forever()
-#vim: tabstop=3 expandtab shiftwidth=3 softtabstop=3 background=dark
+#vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 background=dark
 

roles/captiveportal/templates/checkurls

@@ -15,8 +15,11 @@ teredo.ipv6.microsoft.com
 teredo.ipv6.microsoft.com.nsatc.net
 captive.apple.com
 init-p01st.push.apple.com
-mtalk.google.com
 connectivitycheck.android.com
-alt7-mtalk.google.com
+www.google.com
+mtalk.google.com
+alt4-mtalk.google.com
 alt6-mtalk.google.com
+alt7-mtalk.google.com
+people-pa.googleapis.com
 captive.lan

roles/captiveportal/templates/iiab-divert-to-nginx

@@ -0,0 +1,4 @@
+#!/bin/bash -x
+awk '{print("address=/" $1 "/172.18.96.1")}' /opt/iiab/captiveportal/checkurls > /etc/dnsmasq.d/capture
+echo "#following tells windows 7 that captive portal is active" >>/etc/dnsmasq.d/capture
+echo "address=/dns.msftncsi.com/131.107.255.255" >> /etc/dnsmasq.d/capture

roles/captiveportal/templates/iiab-make-cp-servers.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# read list of online portal checkers, make nginx server blocks 
+
+import os
+outstr = ''
+
+#os.chdir('{{ iiab_dir }}/roles/captiveportal/templates')
+os.chdir('/opt/iiab/iiab/roles/captiveportal/templates')
+with open('checkurls','r') as urls:
+   for line in urls:
+      line = line.replace('*','.*')
+      outstr += 'server {\n'
+      outstr += '    listen 80;\n'
+      outstr += '    server_name {};\n'.format(line.strip())
+      outstr += '    location / {\n'
+      outstr += '        proxy_set_header   X-Forwarded-For $remote_addr;\n'
+      outstr += '        proxy_set_header   Host $http_host;\n'
+      outstr += '        proxy_pass         "http://127.0.0.1:9090";\n'
+      outstr += '    }\n' 
+      outstr += '}\n'
+#print(outstr)
+with open('/etc/nginx/sites-available/capture.conf','w') as config:
+   config.write(outstr)
+

roles/captiveportal/templates/uwsgi-captiveportal.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=uWSGI Service
+
+[Service]
+ExecStart=/usr/bin/uwsgi --ini  /opt/iiab/captiveportal/captiveportal.ini
+Restart=always
+RestartSec=5
+KillSignal=SIGQUIT
+Type=notify
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target

roles/nginx/tasks/install.yml

@@ -25,3 +25,11 @@
     - { src: "roles/nginx/templates/server.conf", dest: "/etc/nginx/" }
     - { src: "roles/nginx/templates/nginx.conf", dest: "/etc/nginx/" }
     - { src: 'roles/nginx/templates/ports.conf', dest: '/etc/{{ apache_service }}/' , mode: '0644' }
+    - { src: 'roles/nginx/templates/uwsgi.service', dest: '/etc/systemd/system/' , mode: '0644' }
+
+- name: Let uwsgi running as {{ apache_user }} write log files
+  file: 
+      path: /var/log/uwsgi/app
+      state: directory
+      owner: "{{ apache_user }}"
+

roles/nginx/templates/nginx.conf

@@ -22,9 +22,10 @@ http {
 	tcp_nodelay on;
 	keepalive_timeout 65;
 	types_hash_max_size 2048;
+
 	# server_tokens off;
 
-	# server_names_hash_bucket_size 64;
+	server_names_hash_bucket_size 64;
 	# server_name_in_redirect off;
 
 	include /etc/nginx/mime.types;

roles/nginx/templates/uwsgi.service

@@ -2,7 +2,7 @@
 Description=uWSGI Service
 
 [Service]
-ExecStart=/usr/local/bin/uwsgi --ini /etc/uwsgi/admin_console_wsgi.ini
+ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/apps-enabled/admin-console.ini
 Restart=always
 RestartSec=5
 KillSignal=SIGQUIT

vars/default_vars.yml

@@ -144,9 +144,10 @@ dns_jail_enabled: False
 # Python-based Captive Portal, that @m-anish & @jvonau experimented with in
 # July 2018 (https://github.com/iiab/iiab/pull/870) and that @georgejhunt
 # extensively refined later in 2018 (PRs #1179, #1300, #1327).
-captive_portal_install: False
+captiveportal_install: False
-captive_portal_enabled: False
+captiveportal_enabled: False
-captive_portal_port: 9090
+captiveportal_port: 9090
+captiveportal_splash_page: /
 # In a pinch, disable Captive Portal using instructions in http://FAQ.IIAB.IO
 
 # Bluetooth PAN access to IIAB server

vars/local_vars_big.yml

@@ -82,8 +82,9 @@ dns_jail_enabled: False
 # Python-based Captive Portal, that @m-anish & @jvonau experimented with in
 # July 2018 (https://github.com/iiab/iiab/pull/870) and that @georgejhunt
 # extensively refined later in 2018 (PRs #1179, #1300, #1327).
-captive_portal_install: False
+captiveportal_install: False
-captive_portal_enabled: False
+captiveportal_enabled: False
+captiveportal_splash_page: /
 # In a pinch, disable Captive Portal using instructions in http://FAQ.IIAB.IO
 
 # Bluetooth PAN access to IIAB server

vars/local_vars_medium.yml

@@ -82,8 +82,9 @@ dns_jail_enabled: False
 # Python-based Captive Portal, that @m-anish & @jvonau experimented with in
 # July 2018 (https://github.com/iiab/iiab/pull/870) and that @georgejhunt
 # extensively refined later in 2018 (PRs #1179, #1300, #1327).
-captive_portal_install: False
+captiveportal_install: False
-captive_portal_enabled: False
+captiveportal_enabled: False
+captiveportal_splash_page: /
 # In a pinch, disable Captive Portal using instructions in http://FAQ.IIAB.IO
 
 # Bluetooth PAN access to IIAB server

vars/local_vars_min.yml

@@ -82,8 +82,9 @@ dns_jail_enabled: False
 # Python-based Captive Portal, that @m-anish & @jvonau experimented with in
 # July 2018 (https://github.com/iiab/iiab/pull/870) and that @georgejhunt
 # extensively refined later in 2018 (PRs #1179, #1300, #1327).
-captive_portal_install: False
+captiveportal_install: False
-captive_portal_enabled: False
+captiveportal_enabled: False
+captiveportal_splash_page: /
 # In a pinch, disable Captive Portal using instructions in http://FAQ.IIAB.IO
 
 # Bluetooth PAN access to IIAB server