1
0
Fork 0
mirror of https://github.com/iiab/iiab.git synced 2025-02-15 04:32:11 +00:00

Merge pull request #148 from iiab/master

sync from iiab/iiab
This commit is contained in:
A Holt 2018-09-05 17:17:58 -04:00 committed by GitHub
commit bc786eb973
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 215 additions and 73 deletions

View file

@ -18,6 +18,8 @@ if flock -n -e 200; then :
echo "Now running iiab-make-kiwix-lib.py"
# write to {{ kiwix_library_xml }}.tmp to minimize kiwix down
# zim map could be out of sync for a few seconds
# using new version that does deltas
cp $KIWIXLIB $KIWIXLIB.tmp
/usr/bin/iiab-make-kiwix-lib.py
{{ systemctl_program }} stop kiwix-serve
rm $KIWIXLIB
@ -27,4 +29,5 @@ else
echo "Can't get wait lock for iiab-make-kiwix-lib.py";
exit 1;
fi
echo 'Finished making Kiwix library.xml'
exit 0

View file

@ -3,6 +3,7 @@
"""
Creates temp library.xml file for kiwix from contents of /zims/content and index
Updated to handle incremental additions and deletions
Author: Tim Moody <tim(at)timmoody(dot)com>
Contributors: Jerry Vonau <jvonau3(at)gmail.com>
@ -19,102 +20,187 @@ import re
import subprocess
import shlex
import ConfigParser
import xml.etree.ElementTree as ET
import argparse
IIAB_PATH='/etc/iiab'
if not IIAB_PATH in sys.path:
sys.path.append(IIAB_PATH)
sys.path.append(IIAB_PATH)
from iiab_env import get_iiab_env
# Config Files
iiab_config_file = "{{ iiab_config_file }}"
# iiab_config_file should be in /etc/iiab/iiab.env
iiab_config_file = "{{ iiab_config_file }}" # nominally /etc/iiab/iiab.ini
# iiab_config_file = "/etc/iiab/iiab.ini" # comment out after testing
IIAB_INI = get_iiab_env('IIAB_INI') # future
if IIAB_INI:
iiab_config_file = IIAB_INI
# Variables that should be read from config file
# All of these variables will be read from config files and recomputed in init()
iiab_zim_path = "{{ iiab_zim_path }}"
zim_path = "/library/zims"
# Later we will append .tmp to file name
kiwix_library_xml = "{{ kiwix_library_xml }}"
iiab_base_path = "{{ iiab_base }}"
iiab_base_path = "/opt/iiab"
kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage"
doc_root = get_iiab_env('WWWROOT')
zim_version_idx = doc_root + "/common/assets/zim_version_idx.json"
zim_versions = {}
zim_version_idx_dir = doc_root + "/common/assets/"
zim_version_idx_file = "zim_version_idx.json"
old_zim_map = {"bad.zim" : "unparseable name"}
# Working variables
# zim_files - list of zims and possible index from file system
# path_to_array_map - list of zims in current library.xml with array index number (for delete)
zim_versions = {} # map of zim's generic name to version installed, e.g. wikipedia_es_all to wikipedia_es_all_2017-01
def main():
"""Server routine"""
global kiwix_library_xml
global zim_path
global zim_version_idx_dir
global zim_version_idx_file
init()
kiwix_library_xml += '.tmp' # write to temp file
args = parse_args()
if args.device: # allow override of path
zim_path = args.device + zim_path
zim_version_idx_dir = args.device + zim_version_idx_dir
# remove existing file
try:
os.remove(kiwix_library_xml)
except OSError:
pass
kiwix_library_xml = zim_path + "/library.xml"
if not args.no_tmp: # don't append .tmp
kiwix_library_xml += ".tmp"
# add each file in /library/zims/content with corresponding index
# only add a single .zim for each .zimxx file
# remove existing file if force
if args.force:
try:
os.remove(kiwix_library_xml)
except OSError:
pass
zims_installed = {}
path_to_array_map = {}
else:
zims_installed, path_to_array_map = read_library_xml(kiwix_library_xml)
zim_files = get_zim_list(zim_path)
# Remove zims not in file system from library.xml
remove_list_str = ""
for item in path_to_array_map:
if item not in zim_files:
remove_list_str += str(path_to_array_map[item]) + " "
if remove_list_str:
rem_libr_xml(remove_list_str)
# Add zims from file system that are not in library.xml
for item in zim_files:
if item not in path_to_array_map:
add_libr_xml(kiwix_library_xml, zim_path, item, zim_files[item])
# Write Version Map
if os.path.isdir(zim_version_idx_dir):
with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp:
json.dump(zim_versions, fp)
else:
print zim_version_idx_dir + " not found."
sys.exit()
def get_zim_list(path):
files_processed = {}
content = iiab_zim_path + "/content/"
index = iiab_zim_path + "/index/"
zim_list = []
content = path + "/content/"
index = path + "/index/"
flist = os.listdir(content)
flist.sort()
for filename in flist:
zimpos = filename.find(".zim")
if zimpos != -1:
filename = filename[:zimpos]
if filename not in files_processed:
files_processed[filename] = True
zimname = "content/" + filename + ".zim"
zimidx = "index/" + filename + ".zim.idx"
if zimname not in files_processed:
if not os.path.isdir (path + "/" + zimidx): # only declare index if exists (could be embedded)
zimidx = None
files_processed[zimname] = zimidx
zimname = content + filename + ".zim"
zimidx = index + filename + ".zim.idx"
command = kiwix_manage + " " + kiwix_library_xml + " add " + zimname
if os.path.isdir (zimidx): # only declare index if exists (could be embedded)
command += " -i " + zimidx
#print command
args = shlex.split(command)
try:
outp = subprocess.check_output(args)
if filename in old_zim_map: # handle old names that don't parse
wiki_name = old_zim_map[filename]
else:
ulpos = filename.rfind("_")
# but gutenberg don't - future maybe put in old_zim_map (en and fr, but instance dates may change)
if "gutenberg_" in filename:
ulpos = filename[:ulpos].rfind("_")
wiki_name = filename[:ulpos]
zim_versions[wiki_name] = filename # if there are multiples, last should win
return files_processed
# create map of generic zim name to actual, assumes pattern of <name>_<yyyy-mm>
# all current files follow this pattern, but some older ones, no longer in the catalog, do not
def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv
kiwix_exclude_attr.append("id") # don't include id
kiwix_exclude_attr.append("favicon") # don't include large favicon
zims_installed = {}
path_to_array_map = {}
try:
tree = ET.parse(lib_xml_file)
root = tree.getroot()
xml_item_no = 0
for child in root:
xml_item_no += 1 # hopefully this is the array number
attributes = {}
if 'id' not in child.attrib: # is this necessary? implies there are records with no book id which would break index for removal
print "xml record missing Book Id"
id = child.attrib['id']
for attr in child.attrib:
if attr not in kiwix_exclude_attr:
attributes[attr] = child.attrib[attr] # copy if not id or in exclusion list
zims_installed[id] = attributes
path_to_array_map[child.attrib['path']] = xml_item_no
except IOError:
zims_installed = {}
return zims_installed, path_to_array_map
if filename in old_zim_map: # handle old names that don't parse
wiki_name = old_zim_map[filename]
else:
ulpos = filename.rfind("_")
# but gutenberg don't - future maybe put in old_zim_map (en and fr, but instance dates may change)
if "gutenberg_" in filename:
ulpos = filename[:ulpos].rfind("_")
wiki_name = filename[:ulpos]
def rem_libr_xml(list_str):
command = kiwix_manage + " " + kiwix_library_xml + " remove " + list_str
print command
args = shlex.split(command)
zim_versions[wiki_name] = filename # if there are multiples, last should win
outp = subprocess.check_output(args)
except: #skip things that don't work
print 'skipping ' + filename
pass
def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx):
command = kiwix_manage + " " + kiwix_library_xml + " add " + zim_path + "/" + zimname
if zimidx:
command += " -i " + zim_path + "/" + zimidx
print command
args = shlex.split(command)
try:
outp = subprocess.check_output(args)
with open(zim_version_idx, 'w') as fp:
json.dump(zim_versions, fp)
sys.exit()
except: #skip things that don't work
print 'skipping ' + filename
pass
def init():
global iiab_base_path
global iiab_zim_path
global zim_path
global kiwix_library_xml
global kiwix_manage
config = ConfigParser.SafeConfigParser()
config.read(iiab_config_file)
iiab_base_path = config.get('location','iiab_base')
iiab_zim_path = config.get('kiwix','iiab_zim_path')
zim_path = config.get('kiwix','iiab_zim_path')
kiwix_library_xml = config.get('kiwix','kiwix_library_xml')
kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage"
def parse_args():
parser = argparse.ArgumentParser(description="Create library.xml for Kiwix.")
parser.add_argument("--device", help="no trailing /. change the target device from internal storage to something else like /media/usb0")
parser.add_argument("--no_tmp", help="don't append .tmp to the library.xml name", action="store_true")
parser.add_argument("-f", "--force", help="force complete rebuild of library.xml", action="store_true")
parser.add_argument("-v", "--verbose", help="Print messages.", action="store_true")
return parser.parse_args()
# Now start the application
if __name__ == "__main__":

View file

@ -54,7 +54,8 @@
with_items:
- /etc/openvpn/keys
- /etc/openvpn/scripts
- /usr/lib/iiab # For executable up_wan. Comment out in future? Might still be relevant for CentOS but unused for ~2 years as of August 2018.
# Obsolete & unused for ~2 years as of August 2018:
#- /usr/lib/iiab
- name: Configure OpenVPN (BACKS UP FILES IF CHANGED)
template:
@ -77,8 +78,8 @@
- { src: 'openvpn_handle.j2', dest: '/etc/iiab/openvpn_handle', mode: '0644' }
# Comment out in future? Not recommended as of August 2018:
- { src: 'iiab-handle.j2', dest: '/usr/bin/iiab-handle', mode: '0755' }
# Comment out in future? Might still be relevant for CentOS but unused for ~2 years as of August 2018:
- { src: 'up_wan', dest: '/usr/lib/iiab/up_wan', mode: '0755' }
# Obsolete & unused for ~2 years as of August 2018:
# - { src: 'up_wan', dest: '/usr/lib/iiab/up_wan', mode: '0755' }
# Obsolete & unused for ~2 years as of August 2018:
#- { src: 'start.j2', dest: '/usr/lib/iiab/start', mode: '0755' }
# Obsolete & unused for ~2 years as of August 2018:
@ -86,6 +87,18 @@
# Obsolete & unused for ~2 years as of August 2018:
#- { src: 'iiab-vpn.j2', dest: '/usr/bin/iiab-vpn', mode: '0755' }
- name: Create iiab-vpn-on (symlink to iiab-remote-on for now)
file:
src: /usr/bin/iiab-remote-on
path: /usr/bin/iiab-vpn-on
state: link
- name: Create iiab-vpn-off (symlink to iiab-remote-off for now)
file:
src: /usr/bin/iiab-remote-off
path: /usr/bin/iiab-vpn-off
state: link
# up_wan was being installed twice (also above) and was unused for ~2 years
# as of August 2018: (see 15-openvpn below)
#- name: Put up_wan in place (debuntu)
@ -97,11 +110,12 @@
# Comment out in future? Contained serious bug (15-openvpn called
# up-wan instead of up_wan in /usr/lib/iiab/ as of August 2018) so
# evidently unused for ~2 years:
- name: Put dispatcher up for NM (not debuntu)
- name: Install NM dispatcher.d (for older OS's only, where OpenVPN doesn't auto-start openvpn@xscenet)
template:
src: 15-openvpn
dest: /etc/NetworkManager/dispatcher.d/
when: not is_debuntu # SHOULD THIS CONDITION ACT ON THE PRESENCE OF NETWORKMANAGER? e.g. some Ubuntu's use NM, others don't.
#when: not is_debuntu # CONDITION APPEARS TOO BROAD
when: False # ADD/ITEMIZE ANY OS'S HERE, WHERE TRULY NEC (e.g. older CentOS, if running older OpenVPN?)
# Was obsolete/unused for ~2 years as of August 2018: (replaced by /etc/openvpn/xscenet.conf)
#- name: Check for manually configured OpenVPN tunnel
@ -137,11 +151,15 @@
# 2018-09-02: OpenVPN had been starting tunnels by accident after reboot,
# with new IIAB installs. Fix below (https://github.com/iiab/iiab/pull/1079)
# changes most all instances below from PARENT service "openvpn@xscenet" to
# CHILD service "openpvn". See these 2 critical files to understand why:
# changes most all instances below from CHILD service "openvpn@xscenet" to
# PARENT service "openpvn". See these critical files to understand why:
#
# /etc/default/openvpn
# /etc/openvpn/xscenet.conf
# /etc/default/openvpn implies AUTOSTART="all"
# /etc/init.d/openvpn has AUTOSTART="all"
# /etc/openvpn/xscenet.conf our VPN connection
# /etc/network/if-up.d/openvpn appears to auto-start xscenet.conf
# /lib/systemd/systemd-sysv-install sets /etc/rc*.d/S|K01openvpn
# e.g. when "systemctl enable openvpn"
- name: Enable & (Re)Start PARENT service openvpn, which (re)starts CHILD service openvpn@xscenet (& actual tunnel)
systemd:
@ -154,6 +172,7 @@
- name: Enable hourly cron job for OpenVPN (starts CHILD service openvpn@xscenet, typically for CentOS only?)
lineinfile:
path: /etc/crontab
# CONSIDER "restart" not just "start" if something stronger is confirmed needed?
line: "25 * * * * root (/usr/bin/systemctl start openvpn@xscenet.service) > /dev/null"
when: openvpn_enabled and openvpn_cron_enabled

View file

@ -1,5 +1,6 @@
#!/bin/bash
# Not really used as of August 2018, but perhaps can be revived for CentOS etc
# Not used as of August 2018: parent service "openvpn" reliably auto-starts child service "openpn@xscenet" on OS's in common use
# But could be revived for older CentOS etc?
export LC_ALL=C
@ -14,7 +15,11 @@ if [ "$2" = "up" ]; then
sleep 2
/sbin/ip route list dev "$1" | grep -q '^default' &&
# restart the services
systemctl -q is-enabled openvpn@xscenet.service && /usr/lib/iiab/up_wan
#systemctl -q is-enabled openvpn@xscenet.service && /usr/lib/iiab/up_wan
# EQUIVALENTLY:
systemctl is-enabled openvpn && pgrep openvpn && systemctl start openvpn@xscenet
# OR EQUIVALENTLY:
# systemctl is-enabled openvpn && systemctl is-active openvpn && systemctl start openvpn@xscenet
fi
# we added this to prevent logs from filling with openvpn errors

View file

@ -13,8 +13,9 @@ if [ -f /etc/iiab/openvpn_handle ]; then
# /etc/iiab/openvpn_handle "obligatory" (EMPTY STRING "" IS TOLERATED, IN WHICH
# CASE OPENVPN SERVER TRIES TO USE /etc/iiab/uuid BELOW, IN LIEU OF HANDLE...)
# CLARIF: "systemctl restart openvpn@xscenet" still runs even if the above is
# defied. e.g. if an implementer deletes /etc/iiab/openvpn_handle by accident.
# CLARIF: "systemctl restart openvpn" still works tolerably even if the above
# is defied, auto-starting child service openvpn@xscenet per usual
# (e.g. if /etc/iiab/openvpn_handle is deleted by accident!)
#else
# # Option #3: Dangerous to invoke hypothetical variables :(

View file

@ -1,5 +1,5 @@
#!/bin/bash
# Interactive script (over)writes /etc/iiab/openvpn_handle file, identifying client to server
# DEPRECATED interactive script (over)writes /etc/iiab/openvpn_handle file, identifying client to server
echo -e '\nCORRECT METHOD: CHANGE VARIABLE openvpn_handle IN /etc/iiab/local_vars.yml'
echo -e 'THEN RUN "cd /opt/iiab/iiab" THEN "./runrole openvpn"\n'
@ -16,3 +16,5 @@ else
echo $ans > /etc/iiab/openvpn_handle
fi
{{ systemctl_program }} restart openvpn@xscenet
# This would also work: (but would bounce all VPN connections, if others exist, causing unnec disruption if so)
#{{ systemctl_program }} restart openvpn

View file

@ -1,17 +1,29 @@
#!/bin/bash
# script to turn on openvpn
# do nothing if it is not installed
# /usr/bin/iiab-remote-off should fully turn off multiple remote support
# services like OpenVPN and others, to reduce risk of remote attacks.
echo -e '\nWARNING: To disable OpenVPN long-term, it'"'"'s recommended you:\n'
echo -e '1) Set this variable in /etc/local/local_vars.yml'
echo -e ' openvpn_enabled: False\n'
echo -e '2) Run:'
echo -e ' cd /opt/iiab/iiab'
echo -e ' sudo ./runrole openvpn\n'
# Do nothing if OpenVPN not installed
which openvpn
if [ $? -ne 0 ]; then
echo 'Cannot find the OpenVPN program (openvpn).'
exit 1
fi
systemctl disable openvpn@xscenet.service
systemctl stop openvpn@xscenet.service
systemctl disable openvpn
systemctl stop openvpn
sleep 5
ps -e|grep vpn
ps -e | grep openvpn # 2018-09-05: "ps -e | grep vpn" no longer works (nor would "pgrep vpn") when invoked from iiab-vpn-off (as filename itself causes [multiple] "vpn" instances to appear in process list!)
if [ $? -eq 0 ]; then
echo OpenVPN failed to stop.
else

View file

@ -1,14 +1,27 @@
#!/bin/bash
# script to turn on openvpn
# do nothing if it is not installed
# /usr/bin/iiab-remote-on should turn on multiple remote support services like
# OpenVPN and others, for remote support, so they work even after reboot.
echo -e '\nWARNING: To enable OpenVPN long-term, it'"'"'s recommended you:\n'
echo -e '1) Set these variables in /etc/local/local_vars.yml'
echo -e ' openvpn_install: True'
echo -e ' openvpn_enabled: True\n'
echo -e '2) Run:'
echo -e ' cd /opt/iiab/iiab'
echo -e ' sudo ./runrole openvpn\n'
# Do nothing if OpenVPN not installed
which openvpn
if [ $? -ne 0 ]; then
echo 'Cannot find the OpenVPN program (openvpn).'
exit 1
fi
systemctl enable openvpn@xscenet.service
systemctl start openvpn@xscenet.service
systemctl enable openvpn
systemctl start openvpn
sleep 5
ping -c 2 {{ openvpn_server_virtual_ip }} # 10.8.0.1

View file

@ -1,5 +1,6 @@
#!/bin/bash
# Not really used as of August 2018, but perhaps can be revived for CentOS etc
# Not used as of August 2018: parent service "openvpn" reliably auto-starts child service "openpn@xscenet" on OS's in common use
# But could be revived for older CentOS etc?
# If the wan has recently come up, see if we need to start openvpn