Merge pull request #148 from iiab/master

sync from iiab/iiab

roles/kiwix/templates/iiab-make-kiwix-lib

@@ -18,6 +18,8 @@ if flock -n -e 200; then :
   echo "Now running iiab-make-kiwix-lib.py"
   # write to {{ kiwix_library_xml }}.tmp to minimize kiwix down
   # zim map could be out of sync for a few seconds
+  # using new version that does deltas
+  cp $KIWIXLIB $KIWIXLIB.tmp
   /usr/bin/iiab-make-kiwix-lib.py
   {{ systemctl_program }} stop kiwix-serve
   rm $KIWIXLIB
@@ -27,4 +29,5 @@ else
   echo "Can't get wait lock for iiab-make-kiwix-lib.py";
   exit 1;
 fi
+echo 'Finished making Kiwix library.xml'
 exit 0

roles/kiwix/templates/iiab-make-kiwix-lib.py

@@ -3,6 +3,7 @@
 """
 
    Creates temp library.xml file for kiwix from contents of /zims/content and index
+   Updated to handle incremental additions and deletions
 
    Author: Tim Moody <tim(at)timmoody(dot)com>
    Contributors: Jerry Vonau <jvonau3(at)gmail.com>
@@ -19,102 +20,187 @@ import re
 import subprocess
 import shlex
 import ConfigParser
+import xml.etree.ElementTree as ET
+import argparse
+
 IIAB_PATH='/etc/iiab'
 if not IIAB_PATH in sys.path:
-   sys.path.append(IIAB_PATH)
+    sys.path.append(IIAB_PATH)
 from iiab_env import get_iiab_env
 
 # Config Files
-iiab_config_file = "{{ iiab_config_file }}"
+# iiab_config_file should be in /etc/iiab/iiab.env
+iiab_config_file = "{{ iiab_config_file }}" # nominally /etc/iiab/iiab.ini
+# iiab_config_file = "/etc/iiab/iiab.ini" # comment out after testing
+
+IIAB_INI = get_iiab_env('IIAB_INI') # future
+if IIAB_INI:
+    iiab_config_file = IIAB_INI
 
 # Variables that should be read from config file
 # All of these variables will be read from config files and recomputed in init()
-iiab_zim_path = "{{ iiab_zim_path }}"
+zim_path = "/library/zims"
 
-# Later we will append .tmp to file name
+iiab_base_path = "/opt/iiab"
-kiwix_library_xml = "{{ kiwix_library_xml }}"
-
-iiab_base_path = "{{ iiab_base }}"
 kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage"
 doc_root = get_iiab_env('WWWROOT')
-zim_version_idx = doc_root + "/common/assets/zim_version_idx.json"
+zim_version_idx_dir = doc_root + "/common/assets/"
-zim_versions = {}
+zim_version_idx_file = "zim_version_idx.json"
+
 old_zim_map = {"bad.zim" : "unparseable name"}
 
+# Working variables
+# zim_files - list of zims and possible index from file system
+# path_to_array_map - list of zims in current library.xml with array index number (for delete)
+zim_versions = {} # map of zim's generic name to version installed, e.g. wikipedia_es_all to wikipedia_es_all_2017-01
+
 def main():
     """Server routine"""
     global kiwix_library_xml
+    global zim_path
+    global zim_version_idx_dir
+    global zim_version_idx_file
+
     init()
-    kiwix_library_xml += '.tmp' # write to temp file    
+    args = parse_args()
+    if args.device: # allow override of path
+        zim_path = args.device + zim_path
+        zim_version_idx_dir = args.device + zim_version_idx_dir
 
-    # remove existing file
+    kiwix_library_xml = zim_path + "/library.xml"
-    try:
+    if not args.no_tmp: # don't append .tmp
-        os.remove(kiwix_library_xml)
+        kiwix_library_xml += ".tmp"
-    except OSError:
-        pass
 
-    # add each file in /library/zims/content with corresponding index
+    # remove existing file if force
-    # only add a single .zim for each .zimxx file
+    if args.force:
+        try:
+            os.remove(kiwix_library_xml)
+        except OSError:
+            pass
+        zims_installed = {}
+        path_to_array_map = {}
+    else:
+          zims_installed, path_to_array_map = read_library_xml(kiwix_library_xml)
 
+    zim_files = get_zim_list(zim_path)
+
+    # Remove zims not in file system from library.xml
+    remove_list_str = ""
+    for item in path_to_array_map:
+          if item not in zim_files:
+              remove_list_str += str(path_to_array_map[item]) + " "
+    if remove_list_str:
+        rem_libr_xml(remove_list_str)
+
+    # Add zims from file system that are not in library.xml
+    for item in zim_files:
+          if item not in path_to_array_map:
+              add_libr_xml(kiwix_library_xml, zim_path, item, zim_files[item])
+
+    # Write Version Map
+    if os.path.isdir(zim_version_idx_dir):
+        with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp:
+            json.dump(zim_versions, fp)
+    else:
+        print zim_version_idx_dir + " not found."
+    sys.exit()
+
+def get_zim_list(path):
     files_processed = {}
-    content = iiab_zim_path + "/content/"
+    zim_list = []
-    index = iiab_zim_path + "/index/"
+    content = path + "/content/"
-
+    index = path + "/index/"
     flist = os.listdir(content)
     flist.sort()
     for filename in flist:
         zimpos = filename.find(".zim")
         if zimpos != -1:
             filename = filename[:zimpos]
-            if filename not in files_processed:
+            zimname = "content/" + filename + ".zim"
-                files_processed[filename] = True
+            zimidx = "index/" + filename + ".zim.idx"
+            if zimname not in files_processed:
+                if not os.path.isdir (path + "/" + zimidx): # only declare index if exists (could be embedded)
+                    zimidx = None
+                files_processed[zimname] = zimidx
                 zimname = content + filename + ".zim"
                 zimidx = index + filename + ".zim.idx"
-                command = kiwix_manage + " " + kiwix_library_xml + " add " + zimname
+                if filename in old_zim_map: # handle old names that don't parse
-                if os.path.isdir (zimidx): # only declare index if exists (could be embedded)
+                    wiki_name = old_zim_map[filename]
-                    command += " -i " + zimidx
+                else:
-                #print command
+                    ulpos = filename.rfind("_")
-                args = shlex.split(command)
+                    # but gutenberg don't - future maybe put in old_zim_map (en and fr, but instance dates may change)
-                try:
+                    if "gutenberg_" in filename:
-                    outp = subprocess.check_output(args)
+                        ulpos = filename[:ulpos].rfind("_")
+                    wiki_name = filename[:ulpos]
+                zim_versions[wiki_name] = filename # if there are multiples, last should win
+    return files_processed
 
-                    # create map of generic zim name to actual, assumes pattern of <name>_<yyyy-mm>
+def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv
-                    # all current files follow this pattern, but some older ones, no longer in the catalog, do not
+    kiwix_exclude_attr.append("id") # don't include id
+    kiwix_exclude_attr.append("favicon") # don't include large favicon
+    zims_installed = {}
+    path_to_array_map = {}
+    try:
+        tree = ET.parse(lib_xml_file)
+        root = tree.getroot()
+        xml_item_no = 0
+        for child in root:
+            xml_item_no += 1 # hopefully this is the array number
+            attributes = {}
+            if 'id' not in child.attrib: # is this necessary? implies there are records with no book id which would break index for removal
+                  print "xml record missing Book Id"
+            id = child.attrib['id']
+            for attr in child.attrib:
+                if attr not in kiwix_exclude_attr:
+                    attributes[attr] = child.attrib[attr] # copy if not id or in exclusion list
+            zims_installed[id] = attributes
+            path_to_array_map[child.attrib['path']] = xml_item_no
+    except IOError:
+        zims_installed = {}
+    return zims_installed, path_to_array_map
 
-                    if filename in old_zim_map: # handle old names that don't parse
+def rem_libr_xml(list_str):
-                        wiki_name = old_zim_map[filename]
+    command = kiwix_manage + " " + kiwix_library_xml + " remove " + list_str
-                    else:
+    print command
-                        ulpos = filename.rfind("_")
+    args = shlex.split(command)
-                        # but gutenberg don't - future maybe put in old_zim_map (en and fr, but instance dates may change)
-                        if "gutenberg_" in filename:
-                            ulpos = filename[:ulpos].rfind("_")
-                        wiki_name = filename[:ulpos]
 
-                    zim_versions[wiki_name] = filename # if there are multiples, last should win
+    outp = subprocess.check_output(args)
 
-                except: #skip things that don't work
+def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx):
-                    print 'skipping ' + filename
+    command = kiwix_manage + " " + kiwix_library_xml + " add " + zim_path + "/" + zimname
-                    pass
+    if zimidx:
+          command += " -i " + zim_path + "/" + zimidx
+    print command
+    args = shlex.split(command)
+    try:
+        outp = subprocess.check_output(args)
 
-    with open(zim_version_idx, 'w') as fp:
+    except: #skip things that don't work
-        json.dump(zim_versions, fp)
+        print 'skipping ' + filename
-
+        pass
-    sys.exit()
 
 def init():
 
     global iiab_base_path
-    global iiab_zim_path
+    global zim_path
     global kiwix_library_xml
     global kiwix_manage
 
     config = ConfigParser.SafeConfigParser()
     config.read(iiab_config_file)
     iiab_base_path = config.get('location','iiab_base')
-    iiab_zim_path = config.get('kiwix','iiab_zim_path')
+    zim_path = config.get('kiwix','iiab_zim_path')
     kiwix_library_xml = config.get('kiwix','kiwix_library_xml')
     kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage"
 
+def parse_args():
+    parser = argparse.ArgumentParser(description="Create library.xml for Kiwix.")
+    parser.add_argument("--device", help="no trailing /. change the target device from internal storage to something else like /media/usb0")
+    parser.add_argument("--no_tmp", help="don't append .tmp to the library.xml name", action="store_true")
+    parser.add_argument("-f", "--force", help="force complete rebuild of library.xml", action="store_true")
+    parser.add_argument("-v", "--verbose", help="Print messages.", action="store_true")
+    return parser.parse_args()
+
 # Now start the application
 
 if __name__ == "__main__":

roles/openvpn/tasks/main.yml

@@ -54,7 +54,8 @@
   with_items:
     - /etc/openvpn/keys
     - /etc/openvpn/scripts
-    - /usr/lib/iiab    # For executable up_wan.  Comment out in future?  Might still be relevant for CentOS but unused for ~2 years as of August 2018.
+    # Obsolete & unused for ~2 years as of August 2018:
+    #- /usr/lib/iiab
 
 - name: Configure OpenVPN (BACKS UP FILES IF CHANGED)
   template:
@@ -77,8 +78,8 @@
     - { src: 'openvpn_handle.j2', dest: '/etc/iiab/openvpn_handle', mode: '0644' }
     # Comment out in future?  Not recommended as of August 2018:
     - { src: 'iiab-handle.j2', dest: '/usr/bin/iiab-handle', mode: '0755' }
-    # Comment out in future?  Might still be relevant for CentOS but unused for ~2 years as of August 2018:
+    # Obsolete & unused for ~2 years as of August 2018:
-    - { src: 'up_wan', dest: '/usr/lib/iiab/up_wan', mode: '0755' }
+    # - { src: 'up_wan', dest: '/usr/lib/iiab/up_wan', mode: '0755' }
     # Obsolete & unused for ~2 years as of August 2018:
     #- { src: 'start.j2', dest: '/usr/lib/iiab/start', mode: '0755' }
     # Obsolete & unused for ~2 years as of August 2018:
@@ -86,6 +87,18 @@
     # Obsolete & unused for ~2 years as of August 2018:
     #- { src: 'iiab-vpn.j2', dest: '/usr/bin/iiab-vpn', mode: '0755' }
 
+- name: Create iiab-vpn-on (symlink to iiab-remote-on for now)
+  file:
+    src: /usr/bin/iiab-remote-on
+    path: /usr/bin/iiab-vpn-on
+    state: link
+
+- name: Create iiab-vpn-off (symlink to iiab-remote-off for now)
+  file:
+    src: /usr/bin/iiab-remote-off
+    path: /usr/bin/iiab-vpn-off
+    state: link
+
 # up_wan was being installed twice (also above) and was unused for ~2 years
 # as of August 2018: (see 15-openvpn below)
 #- name: Put up_wan in place (debuntu)
@@ -97,11 +110,12 @@
 # Comment out in future?  Contained serious bug (15-openvpn called
 # up-wan instead of up_wan in /usr/lib/iiab/ as of August 2018) so
 # evidently unused for ~2 years:
-- name: Put dispatcher up for NM (not debuntu)
+- name: Install NM dispatcher.d (for older OS's only, where OpenVPN doesn't auto-start openvpn@xscenet)
   template:
     src: 15-openvpn
     dest: /etc/NetworkManager/dispatcher.d/
-  when: not is_debuntu    # SHOULD THIS CONDITION ACT ON THE PRESENCE OF NETWORKMANAGER?  e.g. some Ubuntu's use NM, others don't.
+  #when: not is_debuntu    # CONDITION APPEARS TOO BROAD
+  when: False              # ADD/ITEMIZE ANY OS'S HERE, WHERE TRULY NEC (e.g. older CentOS, if running older OpenVPN?)
 
 # Was obsolete/unused for ~2 years as of August 2018: (replaced by /etc/openvpn/xscenet.conf)
 #- name: Check for manually configured OpenVPN tunnel
@@ -137,11 +151,15 @@
 
 # 2018-09-02: OpenVPN had been starting tunnels by accident after reboot,
 # with new IIAB installs.  Fix below (https://github.com/iiab/iiab/pull/1079)
-# changes most all instances below from PARENT service "openvpn@xscenet" to
+# changes most all instances below from CHILD service "openvpn@xscenet" to
-# CHILD service "openpvn".  See these 2 critical files to understand why:
+# PARENT service "openpvn".  See these critical files to understand why:
 #
-#    /etc/default/openvpn
+#    /etc/default/openvpn               implies AUTOSTART="all"
-#    /etc/openvpn/xscenet.conf
+#    /etc/init.d/openvpn                has AUTOSTART="all"
+#    /etc/openvpn/xscenet.conf          our VPN connection
+#    /etc/network/if-up.d/openvpn       appears to auto-start xscenet.conf
+#    /lib/systemd/systemd-sysv-install  sets /etc/rc*.d/S|K01openvpn
+#                                       e.g. when "systemctl enable openvpn"
 
 - name: Enable & (Re)Start PARENT service openvpn, which (re)starts CHILD service openvpn@xscenet (& actual tunnel)
   systemd:
@@ -154,6 +172,7 @@
 - name: Enable hourly cron job for OpenVPN (starts CHILD service openvpn@xscenet, typically for CentOS only?)
   lineinfile:
     path: /etc/crontab
+    # CONSIDER "restart" not just "start" if something stronger is confirmed needed?
     line: "25 *  *  *  * root (/usr/bin/systemctl start openvpn@xscenet.service) > /dev/null"
   when: openvpn_enabled and openvpn_cron_enabled
 

roles/openvpn/templates/15-openvpn

@@ -1,5 +1,6 @@
 #!/bin/bash
-# Not really used as of August 2018, but perhaps can be revived for CentOS etc
+# Not used as of August 2018: parent service "openvpn" reliably auto-starts child service "openpn@xscenet" on OS's in common use
+# But could be revived for older CentOS etc?
 
 export LC_ALL=C
 
@@ -14,7 +15,11 @@ if [ "$2" = "up" ]; then
     sleep 2
     /sbin/ip route list dev "$1" | grep -q '^default' &&
     # restart the services
-    systemctl -q is-enabled openvpn@xscenet.service && /usr/lib/iiab/up_wan
+    #systemctl -q is-enabled openvpn@xscenet.service && /usr/lib/iiab/up_wan
+    # EQUIVALENTLY:
+    systemctl is-enabled openvpn && pgrep openvpn && systemctl start openvpn@xscenet
+    # OR EQUIVALENTLY:
+    # systemctl is-enabled openvpn && systemctl is-active openvpn && systemctl start openvpn@xscenet
 fi
 
 # we added this to prevent logs from filling with openvpn errors

roles/openvpn/templates/announcer.j2

@@ -13,8 +13,9 @@ if [ -f /etc/iiab/openvpn_handle ]; then
 # /etc/iiab/openvpn_handle "obligatory" (EMPTY STRING "" IS TOLERATED, IN WHICH
 # CASE OPENVPN SERVER TRIES TO USE /etc/iiab/uuid BELOW, IN LIEU OF HANDLE...)
 
-# CLARIF: "systemctl restart openvpn@xscenet" still runs even if the above is
+# CLARIF: "systemctl restart openvpn" still works tolerably even if the above
-# defied.  e.g. if an implementer deletes /etc/iiab/openvpn_handle by accident.
+# is defied, auto-starting child service openvpn@xscenet per usual
+# (e.g. if /etc/iiab/openvpn_handle is deleted by accident!)
 
 #else
 #    # Option #3: Dangerous to invoke hypothetical variables :(

roles/openvpn/templates/iiab-handle.j2

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Interactive script (over)writes /etc/iiab/openvpn_handle file, identifying client to server
+# DEPRECATED interactive script (over)writes /etc/iiab/openvpn_handle file, identifying client to server
 
 echo -e '\nCORRECT METHOD: CHANGE VARIABLE openvpn_handle IN /etc/iiab/local_vars.yml'
 echo -e 'THEN RUN "cd /opt/iiab/iiab" THEN "./runrole openvpn"\n'
@@ -16,3 +16,5 @@ else
     echo $ans > /etc/iiab/openvpn_handle
 fi
 {{ systemctl_program }} restart openvpn@xscenet
+# This would also work: (but would bounce all VPN connections, if others exist, causing unnec disruption if so)
+#{{ systemctl_program }} restart openvpn

roles/openvpn/templates/iiab-remote-off

@@ -1,17 +1,29 @@
 #!/bin/bash
-# script to turn on openvpn
 
-# do nothing if it is not installed
+# /usr/bin/iiab-remote-off should fully turn off multiple remote support
+# services like OpenVPN and others, to reduce risk of remote attacks.
+
+echo -e '\nWARNING: To disable OpenVPN long-term, it'"'"'s recommended you:\n'
+
+echo -e '1) Set this variable in /etc/local/local_vars.yml'
+echo -e '   openvpn_enabled: False\n'
+
+echo -e '2) Run:'
+echo -e '   cd /opt/iiab/iiab'
+echo -e '   sudo ./runrole openvpn\n'
+
+# Do nothing if OpenVPN not installed
 which openvpn
 if [ $? -ne 0 ]; then
     echo 'Cannot find the OpenVPN program (openvpn).'
     exit 1
 fi
-systemctl disable openvpn@xscenet.service
+
-systemctl stop openvpn@xscenet.service
+systemctl disable openvpn
+systemctl stop openvpn
 
 sleep 5
-ps -e|grep vpn
+ps -e | grep openvpn    # 2018-09-05: "ps -e | grep vpn" no longer works (nor would "pgrep vpn") when invoked from iiab-vpn-off (as filename itself causes [multiple] "vpn" instances to appear in process list!)
 if [ $? -eq 0 ]; then
     echo OpenVPN failed to stop.
 else

roles/openvpn/templates/iiab-remote-on.j2

@@ -1,14 +1,27 @@
 #!/bin/bash
-# script to turn on openvpn
 
-# do nothing if it is not installed
+# /usr/bin/iiab-remote-on should turn on multiple remote support services like
+# OpenVPN and others, for remote support, so they work even after reboot.
+
+echo -e '\nWARNING: To enable OpenVPN long-term, it'"'"'s recommended you:\n'
+
+echo -e '1) Set these variables in /etc/local/local_vars.yml'
+echo -e '   openvpn_install: True'
+echo -e '   openvpn_enabled: True\n'
+
+echo -e '2) Run:'
+echo -e '   cd /opt/iiab/iiab'
+echo -e '   sudo ./runrole openvpn\n'
+
+# Do nothing if OpenVPN not installed
 which openvpn
 if [ $? -ne 0 ]; then
     echo 'Cannot find the OpenVPN program (openvpn).'
     exit 1
 fi
-systemctl enable openvpn@xscenet.service
+
-systemctl start openvpn@xscenet.service
+systemctl enable openvpn
+systemctl start openvpn
 
 sleep 5
 ping -c 2 {{ openvpn_server_virtual_ip }}    # 10.8.0.1

roles/openvpn/templates/up_wan.deprecated

@@ -1,5 +1,6 @@
 #!/bin/bash
-# Not really used as of August 2018, but perhaps can be revived for CentOS etc
+# Not used as of August 2018: parent service "openvpn" reliably auto-starts child service "openpn@xscenet" on OS's in common use
+# But could be revived for older CentOS etc?
 
 # If the wan has recently come up, see if we need to start openvpn