From d93e5d8a8709c9e47e0e85badfd555e0e7c547ad Mon Sep 17 00:00:00 2001 From: Tim Moody Date: Tue, 4 Sep 2018 13:49:26 -0400 Subject: [PATCH 01/11] change make-kiwix-lib to only update additions and deletions --- roles/kiwix/templates/iiab-make-kiwix-lib | 3 + roles/kiwix/templates/iiab-make-kiwix-lib.py | 184 ++++++++++++++----- 2 files changed, 138 insertions(+), 49 deletions(-) diff --git a/roles/kiwix/templates/iiab-make-kiwix-lib b/roles/kiwix/templates/iiab-make-kiwix-lib index 5680470f3..f36857064 100644 --- a/roles/kiwix/templates/iiab-make-kiwix-lib +++ b/roles/kiwix/templates/iiab-make-kiwix-lib @@ -18,6 +18,8 @@ if flock -n -e 200; then : echo "Now running iiab-make-kiwix-lib.py" # write to {{ kiwix_library_xml }}.tmp to minimize kiwix down # zim map could be out of sync for a few seconds + # using new version that does deltas + cp $KIWIXLIB $KIWIXLIB.tmp /usr/bin/iiab-make-kiwix-lib.py {{ systemctl_program }} stop kiwix-serve rm $KIWIXLIB @@ -27,4 +29,5 @@ else echo "Can't get wait lock for iiab-make-kiwix-lib.py"; exit 1; fi +echo 'Finished making Kiwix library.xml' exit 0 diff --git a/roles/kiwix/templates/iiab-make-kiwix-lib.py b/roles/kiwix/templates/iiab-make-kiwix-lib.py index d331f194d..6ca92b1c1 100644 --- a/roles/kiwix/templates/iiab-make-kiwix-lib.py +++ b/roles/kiwix/templates/iiab-make-kiwix-lib.py @@ -3,6 +3,7 @@ """ Creates temp library.xml file for kiwix from contents of /zims/content and index + Updated to handle incremental additions and deletions Author: Tim Moody Contributors: Jerry Vonau @@ -19,102 +20,187 @@ import re import subprocess import shlex import ConfigParser +import xml.etree.ElementTree as ET +import argparse + IIAB_PATH='/etc/iiab' if not IIAB_PATH in sys.path: - sys.path.append(IIAB_PATH) + sys.path.append(IIAB_PATH) from iiab_env import get_iiab_env # Config Files -iiab_config_file = "{{ iiab_config_file }}" +# iiab_config_file should be in /etc/iiab/iiab.env +iiab_config_file = "{{ iiab_config_file }}" # nominally /etc/iiab/iiab.ini +# iiab_config_file = "/etc/iiab/iiab.ini" # comment out after testing + +IIAB_INI = get_iiab_env('IIAB_INI') # future +if IIAB_INI: + iiab_config_file = IIAB_INI # Variables that should be read from config file # All of these variables will be read from config files and recomputed in init() -iiab_zim_path = "{{ iiab_zim_path }}" +zim_path = "/library/zims" -# Later we will append .tmp to file name -kiwix_library_xml = "{{ kiwix_library_xml }}" - -iiab_base_path = "{{ iiab_base }}" +iiab_base_path = "/opt/iiab" kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage" doc_root = get_iiab_env('WWWROOT') -zim_version_idx = doc_root + "/common/assets/zim_version_idx.json" -zim_versions = {} +zim_version_idx_dir = doc_root + "/common/assets/" +zim_version_idx_file = "zim_version_idx.json" + old_zim_map = {"bad.zim" : "unparseable name"} +# Working variables +# zim_files - list of zims and possible index from file system +# path_to_array_map - list of zims in current library.xml with array index number (for delete) +zim_versions = {} # map of zim's generic name to version installed, e.g. wikipedia_es_all to wikipedia_es_all_2017-01 + def main(): """Server routine""" global kiwix_library_xml + global zim_path + global zim_version_idx_dir + global zim_version_idx_file + init() - kiwix_library_xml += '.tmp' # write to temp file + args = parse_args() + if args.device: # allow override of path + zim_path = args.device + zim_path + zim_version_idx_dir = args.device + zim_version_idx_dir - # remove existing file - try: - os.remove(kiwix_library_xml) - except OSError: - pass + kiwix_library_xml = zim_path + "/library.xml" + if not args.no_tmp: # don't append .tmp + kiwix_library_xml += ".tmp" - # add each file in /library/zims/content with corresponding index - # only add a single .zim for each .zimxx file + # remove existing file if force + if args.force: + try: + os.remove(kiwix_library_xml) + except OSError: + pass + zims_installed = {} + path_to_array_map = {} + else: + zims_installed, path_to_array_map = read_library_xml(kiwix_library_xml) + zim_files = get_zim_list(zim_path) + + # Remove zims not in file system from library.xml + remove_list_str = "" + for item in path_to_array_map: + if item not in zim_files: + remove_list_str += str(path_to_array_map[item]) + " " + if remove_list_str: + rem_libr_xml(remove_list_str) + + # Add zims from file system that are not in library.xml + for item in zim_files: + if item not in path_to_array_map: + add_libr_xml(kiwix_library_xml, zim_path, item, zim_files[item]) + + # Write Version Map + if os.path.isdir(zim_version_idx_dir): + with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp: + json.dump(zim_versions, fp) + else: + print zim_version_idx_dir + " not found." + sys.exit() + +def get_zim_list(path): files_processed = {} - content = iiab_zim_path + "/content/" - index = iiab_zim_path + "/index/" - + zim_list = [] + content = path + "/content/" + index = path + "/index/" flist = os.listdir(content) flist.sort() for filename in flist: zimpos = filename.find(".zim") if zimpos != -1: filename = filename[:zimpos] - if filename not in files_processed: - files_processed[filename] = True + zimname = "content/" + filename + ".zim" + zimidx = "index/" + filename + ".zim.idx" + if zimname not in files_processed: + if not os.path.isdir (path + "/" + zimidx): # only declare index if exists (could be embedded) + zimidx = None + files_processed[zimname] = zimidx zimname = content + filename + ".zim" zimidx = index + filename + ".zim.idx" - command = kiwix_manage + " " + kiwix_library_xml + " add " + zimname - if os.path.isdir (zimidx): # only declare index if exists (could be embedded) - command += " -i " + zimidx - #print command - args = shlex.split(command) - try: - outp = subprocess.check_output(args) + if filename in old_zim_map: # handle old names that don't parse + wiki_name = old_zim_map[filename] + else: + ulpos = filename.rfind("_") + # but gutenberg don't - future maybe put in old_zim_map (en and fr, but instance dates may change) + if "gutenberg_" in filename: + ulpos = filename[:ulpos].rfind("_") + wiki_name = filename[:ulpos] + zim_versions[wiki_name] = filename # if there are multiples, last should win + return files_processed - # create map of generic zim name to actual, assumes pattern of _ - # all current files follow this pattern, but some older ones, no longer in the catalog, do not +def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv + kiwix_exclude_attr.append("id") # don't include id + kiwix_exclude_attr.append("favicon") # don't include large favicon + zims_installed = {} + path_to_array_map = {} + try: + tree = ET.parse(lib_xml_file) + root = tree.getroot() + xml_item_no = 0 + for child in root: + xml_item_no += 1 # hopefully this is the array number + attributes = {} + if 'id' not in child.attrib: # is this necessary? implies there are records with no book id which would break index for removal + print "xml record missing Book Id" + id = child.attrib['id'] + for attr in child.attrib: + if attr not in kiwix_exclude_attr: + attributes[attr] = child.attrib[attr] # copy if not id or in exclusion list + zims_installed[id] = attributes + path_to_array_map[child.attrib['path']] = xml_item_no + except IOError: + zims_installed = {} + return zims_installed, path_to_array_map - if filename in old_zim_map: # handle old names that don't parse - wiki_name = old_zim_map[filename] - else: - ulpos = filename.rfind("_") - # but gutenberg don't - future maybe put in old_zim_map (en and fr, but instance dates may change) - if "gutenberg_" in filename: - ulpos = filename[:ulpos].rfind("_") - wiki_name = filename[:ulpos] +def rem_libr_xml(list_str): + command = kiwix_manage + " " + kiwix_library_xml + " remove " + list_str + print command + args = shlex.split(command) - zim_versions[wiki_name] = filename # if there are multiples, last should win + outp = subprocess.check_output(args) - except: #skip things that don't work - print 'skipping ' + filename - pass +def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx): + command = kiwix_manage + " " + kiwix_library_xml + " add " + zim_path + "/" + zimname + if zimidx: + command += " -i " + zim_path + "/" + zimidx + print command + args = shlex.split(command) + try: + outp = subprocess.check_output(args) - with open(zim_version_idx, 'w') as fp: - json.dump(zim_versions, fp) - - sys.exit() + except: #skip things that don't work + print 'skipping ' + filename + pass def init(): global iiab_base_path - global iiab_zim_path + global zim_path global kiwix_library_xml global kiwix_manage config = ConfigParser.SafeConfigParser() config.read(iiab_config_file) iiab_base_path = config.get('location','iiab_base') - iiab_zim_path = config.get('kiwix','iiab_zim_path') + zim_path = config.get('kiwix','iiab_zim_path') kiwix_library_xml = config.get('kiwix','kiwix_library_xml') kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage" +def parse_args(): + parser = argparse.ArgumentParser(description="Create library.xml for Kiwix.") + parser.add_argument("--device", help="no trailing /. change the target device from internal storage to something else like /media/usb0") + parser.add_argument("--no_tmp", help="don't append .tmp to the library.xml name", action="store_true") + parser.add_argument("-f", "--force", help="force complete rebuild of library.xml", action="store_true") + parser.add_argument("-v", "--verbose", help="Print messages.", action="store_true") + return parser.parse_args() + # Now start the application if __name__ == "__main__": From 7451e5940061638848bafe57396af51e8d6868dd Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 14:48:48 -0400 Subject: [PATCH 02/11] Update main.yml --- roles/openvpn/tasks/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/roles/openvpn/tasks/main.yml b/roles/openvpn/tasks/main.yml index 3fe9b5945..3000da823 100644 --- a/roles/openvpn/tasks/main.yml +++ b/roles/openvpn/tasks/main.yml @@ -87,6 +87,22 @@ # Obsolete & unused for ~2 years as of August 2018: #- { src: 'iiab-vpn.j2', dest: '/usr/bin/iiab-vpn', mode: '0755' } +# iiab-remote-on is intended to turn on multiple remote support services like +# OpenVPN and others, for remote support, so they work even after reboot. +- name: Create iiab-vpn-on (a symbolic link to iiab-remote-on for now) + file: + src: /usr/bin/iiab-remote-on + path: /usr/bin/iiab-vpn-on + state: link + +# iiab-remote-off is intended to fully turn off multiple remote support +# services like OpenVPN and others, to reduce the risk of remote attacks. +- name: Create iiab-vpn-off (a symbolic link to iiab-remote-off for now) + file: + src: /usr/bin/iiab-remote-off + path: /usr/bin/iiab-vpn-off + state: link + # up_wan was being installed twice (also above) and was unused for ~2 years # as of August 2018: (see 15-openvpn below) #- name: Put up_wan in place (debuntu) From 6e18a5af152c9de55d38f8931f7ca674fd6c1854 Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 15:13:38 -0400 Subject: [PATCH 03/11] longstanding syntax error (bash wouldn't run this on Ubuntu 18.04) --- roles/openvpn/templates/iiab-remote-on.j2 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/openvpn/templates/iiab-remote-on.j2 b/roles/openvpn/templates/iiab-remote-on.j2 index 6f6c8104a..f97bf11e7 100644 --- a/roles/openvpn/templates/iiab-remote-on.j2 +++ b/roles/openvpn/templates/iiab-remote-on.j2 @@ -1,12 +1,15 @@ #!/bin/bash -# script to turn on openvpn -# do nothing if it is not installed +# /usr/bin/iiab-remote-on should turn on multiple remote support services like +# OpenVPN and others, for remote support, so they work even after reboot. + +# Do nothing if OpenVPN not installed which openvpn if [ $? -ne 0 ]; then - echo Cannot find the OpenVPN program (openvpn). + echo 'Cannot find the OpenVPN program (openvpn).' exit 1 fi + systemctl enable openvpn systemctl start openvpn From 96d1eae568a531e12c030f9e7186b6e852946640 Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 15:13:47 -0400 Subject: [PATCH 04/11] longstanding syntax error (bash wouldn't run this on Ubuntu 18.04) --- roles/openvpn/templates/iiab-remote-off | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/roles/openvpn/templates/iiab-remote-off b/roles/openvpn/templates/iiab-remote-off index 991c50f01..953c3d878 100644 --- a/roles/openvpn/templates/iiab-remote-off +++ b/roles/openvpn/templates/iiab-remote-off @@ -1,17 +1,20 @@ #!/bin/bash -# script to turn on openvpn -# do nothing if it is not installed +# /usr/bin/iiab-remote-off is intended to fully turn off multiple remote +# support services like OpenVPN and others, to reduce risk of remote attacks. + +# Do nothing if OpenVPN not installed which openvpn if [ $? -ne 0 ]; then - echo Cannot find the OpenVPN program (openvpn). + echo 'Cannot find the OpenVPN program (openvpn).' exit 1 fi + systemctl disable openvpn systemctl stop openvpn sleep 5 -ps -e|grep vpn +ps -e | grep vpn if [ $? -eq 0 ]; then echo OpenVPN failed to stop. else From f21c0f2fa9f3c41fb09b711a84d8fbd9056464ca Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 15:33:42 -0400 Subject: [PATCH 05/11] Update iiab-remote-off --- roles/openvpn/templates/iiab-remote-off | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/openvpn/templates/iiab-remote-off b/roles/openvpn/templates/iiab-remote-off index 953c3d878..ca1ff41c2 100644 --- a/roles/openvpn/templates/iiab-remote-off +++ b/roles/openvpn/templates/iiab-remote-off @@ -1,7 +1,7 @@ #!/bin/bash -# /usr/bin/iiab-remote-off is intended to fully turn off multiple remote -# support services like OpenVPN and others, to reduce risk of remote attacks. +# /usr/bin/iiab-remote-off should fully turn off multiple remote support +# services like OpenVPN and others, to reduce risk of remote attacks. # Do nothing if OpenVPN not installed which openvpn From 3b2c16d6bf83e4efe210acf3b4ef93d2f2fedd3a Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 15:53:20 -0400 Subject: [PATCH 06/11] Update main.yml --- roles/openvpn/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/openvpn/tasks/main.yml b/roles/openvpn/tasks/main.yml index 3000da823..4f7b7e463 100644 --- a/roles/openvpn/tasks/main.yml +++ b/roles/openvpn/tasks/main.yml @@ -87,7 +87,7 @@ # Obsolete & unused for ~2 years as of August 2018: #- { src: 'iiab-vpn.j2', dest: '/usr/bin/iiab-vpn', mode: '0755' } -# iiab-remote-on is intended to turn on multiple remote support services like +# /usr/bin/iiab-remote-on should turn on multiple remote support services like # OpenVPN and others, for remote support, so they work even after reboot. - name: Create iiab-vpn-on (a symbolic link to iiab-remote-on for now) file: @@ -95,8 +95,8 @@ path: /usr/bin/iiab-vpn-on state: link -# iiab-remote-off is intended to fully turn off multiple remote support -# services like OpenVPN and others, to reduce the risk of remote attacks. +# /usr/bin/iiab-remote-off should fully turn off multiple remote support +# services like OpenVPN and others, to reduce risk of remote attacks. - name: Create iiab-vpn-off (a symbolic link to iiab-remote-off for now) file: src: /usr/bin/iiab-remote-off From 849be849d09b505ef459e5f4943a15fc86224b4f Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 15:56:22 -0400 Subject: [PATCH 07/11] Update iiab-remote-on.j2 --- roles/openvpn/templates/iiab-remote-on.j2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/openvpn/templates/iiab-remote-on.j2 b/roles/openvpn/templates/iiab-remote-on.j2 index f97bf11e7..8a0e6284d 100644 --- a/roles/openvpn/templates/iiab-remote-on.j2 +++ b/roles/openvpn/templates/iiab-remote-on.j2 @@ -3,6 +3,16 @@ # /usr/bin/iiab-remote-on should turn on multiple remote support services like # OpenVPN and others, for remote support, so they work even after reboot. +echo -e 'WARNING: To enable OpenVPN long-term, it'"'"'s recommended you:\n' + +echo -e '1) Set these variables in /etc/local/local_vars.yml' +echo -e ' openvpn_install: True' +echo -e ' openvpn_enabled: True\n' + +echo -e '2) Run:' +echo -e ' cd /opt/iiab/iiab' +echo -e ' sudo ./runrole openvpn\n' + # Do nothing if OpenVPN not installed which openvpn if [ $? -ne 0 ]; then From f126aec86d81381ebe228525bd8a5f715e988dbc Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 15:58:50 -0400 Subject: [PATCH 08/11] Update main.yml --- roles/openvpn/tasks/main.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/roles/openvpn/tasks/main.yml b/roles/openvpn/tasks/main.yml index 4f7b7e463..6efc9cb9b 100644 --- a/roles/openvpn/tasks/main.yml +++ b/roles/openvpn/tasks/main.yml @@ -87,17 +87,13 @@ # Obsolete & unused for ~2 years as of August 2018: #- { src: 'iiab-vpn.j2', dest: '/usr/bin/iiab-vpn', mode: '0755' } -# /usr/bin/iiab-remote-on should turn on multiple remote support services like -# OpenVPN and others, for remote support, so they work even after reboot. -- name: Create iiab-vpn-on (a symbolic link to iiab-remote-on for now) +- name: Create iiab-vpn-on (symlink to iiab-remote-on for now) file: src: /usr/bin/iiab-remote-on path: /usr/bin/iiab-vpn-on state: link -# /usr/bin/iiab-remote-off should fully turn off multiple remote support -# services like OpenVPN and others, to reduce risk of remote attacks. -- name: Create iiab-vpn-off (a symbolic link to iiab-remote-off for now) +- name: Create iiab-vpn-off (symlink to iiab-remote-off for now) file: src: /usr/bin/iiab-remote-off path: /usr/bin/iiab-vpn-off From 224ea76e5a6353d59b233af4c8039f4853122903 Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 16:14:23 -0400 Subject: [PATCH 09/11] Update iiab-remote-on.j2 --- roles/openvpn/templates/iiab-remote-on.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openvpn/templates/iiab-remote-on.j2 b/roles/openvpn/templates/iiab-remote-on.j2 index 8a0e6284d..8771cb94f 100644 --- a/roles/openvpn/templates/iiab-remote-on.j2 +++ b/roles/openvpn/templates/iiab-remote-on.j2 @@ -3,7 +3,7 @@ # /usr/bin/iiab-remote-on should turn on multiple remote support services like # OpenVPN and others, for remote support, so they work even after reboot. -echo -e 'WARNING: To enable OpenVPN long-term, it'"'"'s recommended you:\n' +echo -e '\nWARNING: To enable OpenVPN long-term, it'"'"'s recommended you:\n' echo -e '1) Set these variables in /etc/local/local_vars.yml' echo -e ' openvpn_install: True' From 725283decfbb75ed655cf0cafdcb9ed6478055f4 Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 16:17:02 -0400 Subject: [PATCH 10/11] Update iiab-remote-off --- roles/openvpn/templates/iiab-remote-off | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/openvpn/templates/iiab-remote-off b/roles/openvpn/templates/iiab-remote-off index ca1ff41c2..b7ffc9da2 100644 --- a/roles/openvpn/templates/iiab-remote-off +++ b/roles/openvpn/templates/iiab-remote-off @@ -3,6 +3,15 @@ # /usr/bin/iiab-remote-off should fully turn off multiple remote support # services like OpenVPN and others, to reduce risk of remote attacks. +echo -e '\nWARNING: To disable OpenVPN long-term, it'"'"'s recommended you:\n' + +echo -e '1) Set this variable in /etc/local/local_vars.yml' +echo -e ' openvpn_enabled: False\n' + +echo -e '2) Run:' +echo -e ' cd /opt/iiab/iiab' +echo -e ' sudo ./runrole openvpn\n' + # Do nothing if OpenVPN not installed which openvpn if [ $? -ne 0 ]; then From 1fcece61716e802075d0e4d60ac054d104d65057 Mon Sep 17 00:00:00 2001 From: A Holt Date: Wed, 5 Sep 2018 16:39:34 -0400 Subject: [PATCH 11/11] Update iiab-remote-off --- roles/openvpn/templates/iiab-remote-off | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openvpn/templates/iiab-remote-off b/roles/openvpn/templates/iiab-remote-off index b7ffc9da2..f80377a64 100644 --- a/roles/openvpn/templates/iiab-remote-off +++ b/roles/openvpn/templates/iiab-remote-off @@ -23,7 +23,7 @@ systemctl disable openvpn systemctl stop openvpn sleep 5 -ps -e | grep vpn +ps -e | grep openvpn # 2018-09-05: "ps -e | grep vpn" no longer works (nor would "pgrep vpn") when invoked from iiab-vpn-off (as filename itself causes [multiple] "vpn" instances to appear in process list!) if [ $? -eq 0 ]; then echo OpenVPN failed to stop. else