From 7f9957aa55f9826697783afbff64d7969301001f Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Thu, 10 Jan 2019 14:26:47 -0800 Subject: [PATCH 1/3] 0.3 patch1 (#1377) * expand zim_versions_idx to include menuItem name, mediaCount, articleCount, size * create a stub menuItem if none exists * remember to change consumer of zim_version_idx in iiab-admin * comment out some debugging prints * do not change the name of a variable in iiab.ini * consistent variable names for *_enabled * start work on menus for enabled iiab roles * changes to display cups in home menu * remove reference to admin console which may not be installed * print error message * dict.get errors * handle undefined menuItem" * handle undefined menuItem again * some work on logos * break out the zim_versions_idx routines for use by update_menus in admin-console * remove old menuDef creation code * new variable names for zim_versions_idx * missing the tags data in zim_versions_idx * un-break-apart iiab-make-kiwix-lib.py * return an empty string * make size human_readable * getting console and iiab to work together on menus * add the new zim_date field to zim_versions_idx * get the latest into repo * found lost code iiab-make-kiwix-lib.py * Create Lokole admin user during setup * Add requested content to lokole readme Per iiab/iiab#1293 * Update default password * Add Lokole-IIAB user manual * Update default_vars.yml * Update local_vars_min.yml * Update local_vars_min.yml * Update local_vars_medium.yml * Update local_vars_min.yml * Update default_vars.yml * Update local_vars_big.yml * Update local_vars_medium.yml * Update default_vars.yml * Update default_vars.yml * Update local_vars_min.yml * Update local_vars_medium.yml * Update local_vars_big.yml * Update default_vars.yml * Update default_vars.yml * Update local_vars_big.yml * Update local_vars_medium.yml * Update local_vars_min.yml * Update default_vars.yml * Update local_vars_big.yml * Update local_vars_medium.yml * Update local_vars_min.yml * Change admin username to uppercase * Revert "Lokole: change admin to Admin per IIAB app norms" * Update main.yml * Update README.rst * Update capture-wsgi.py * Update main.yml * Update main.yml * Update default_vars.yml * Update local_vars_big.yml * Update local_vars_medium.yml * Update local_vars_min.yml * Update main.yml * Update main.yml * Update main.yml * Update main.yml * Update default_vars.yml * Update local_vars_big.yml * Update local_vars_medium.yml * Update local_vars_min.yml * Address TODOs in Lokole documentation See https://github.com/ascoderu/opwen-webapp/issues/81 * Update main.yml * Update local_vars_big.yml * Update local_vars_medium.yml * Update local_vars_big.yml * Update local_vars_min.yml * Update default_vars.yml * expand zim_versions_idx to include menuItem name, mediaCount, articleCount, size * create a stub menuItem if none exists * remember to change consumer of zim_version_idx in iiab-admin * comment out some debugging prints * do not change the name of a variable in iiab.ini * consistent variable names for *_enabled * start work on menus for enabled iiab roles * changes to display cups in home menu * remove reference to admin console which may not be installed * print error message * dict.get errors * handle undefined menuItem" * handle undefined menuItem again * some work on logos * break out the zim_versions_idx routines for use by update_menus in admin-console * remove old menuDef creation code * new variable names for zim_versions_idx * missing the tags data in zim_versions_idx * un-break-apart iiab-make-kiwix-lib.py * return an empty string * make size human_readable * getting console and iiab to work together on menus * add the new zim_date field to zim_versions_idx * get the latest into repo * found lost code iiab-make-kiwix-lib.py --- roles/calibre/tasks/main.yml | 2 +- roles/cups/tasks/main.yml | 2 +- roles/elgg/tasks/main.yml | 2 +- roles/kalite/tasks/main.yml | 2 +- roles/kiwix/tasks/kiwix_install.yml | 2 +- roles/kiwix/templates/iiab-make-kiwix-lib.py | 124 ++++++++++++++++-- roles/kolibri/tasks/main.yml | 2 +- .../templates/gateway/iiab-gen-iptables | 2 + roles/sugarizer/tasks/main.yml | 2 +- roles/teamviewer/tasks/main.yml | 4 +- vars/default_vars.yml | 1 + 11 files changed, 123 insertions(+), 22 deletions(-) mode change 100644 => 100755 roles/kiwix/templates/iiab-make-kiwix-lib.py diff --git a/roles/calibre/tasks/main.yml b/roles/calibre/tasks/main.yml index ab16676c1..c2f2239e1 100644 --- a/roles/calibre/tasks/main.yml +++ b/roles/calibre/tasks/main.yml @@ -147,5 +147,5 @@ value: "{{ calibre_dbpath }}" - option: port value: "{{ calibre_port }}" - - option: enabled + - option: calibre_enabled value: "{{ calibre_enabled }}" diff --git a/roles/cups/tasks/main.yml b/roles/cups/tasks/main.yml index e0c9f3cb1..73072a07b 100644 --- a/roles/cups/tasks/main.yml +++ b/roles/cups/tasks/main.yml @@ -75,5 +75,5 @@ value: '"CUPS (Common UNIX Printing System) is a modular printing system that allows a computer to act as a print server. A computer running CUPS is a host that can accept print jobs from client computers, process them, and send them to the appropriate printer."' - option: installed value: "{{ cups_install }}" - - option: enabled + - option: cups_enabled value: "{{ cups_enabled }}" diff --git a/roles/elgg/tasks/main.yml b/roles/elgg/tasks/main.yml index 29b7e34c0..9cab2c00d 100644 --- a/roles/elgg/tasks/main.yml +++ b/roles/elgg/tasks/main.yml @@ -161,5 +161,5 @@ value: '"Elgg is an award-winning social networking engine, delivering the building blocks that enable businesses, schools, universities and associations to create their own fully-featured social networks and applications."' - option: path value: /opt/elgg - - option: enabled + - option: elgg_enabled value: "{{ elgg_enabled }}" diff --git a/roles/kalite/tasks/main.yml b/roles/kalite/tasks/main.yml index 28063b45f..3be716a1b 100644 --- a/roles/kalite/tasks/main.yml +++ b/roles/kalite/tasks/main.yml @@ -51,7 +51,7 @@ value: "{{ kalite_root }}" - option: port value: "{{ kalite_server_port }}" - - option: enabled + - option: kalite_enabled value: "{{ kalite_enabled }}" - option: cron_enabled value: "{{ kalite_cron_enabled }}" diff --git a/roles/kiwix/tasks/kiwix_install.yml b/roles/kiwix/tasks/kiwix_install.yml index fa09e5f00..430e4d73c 100644 --- a/roles/kiwix/tasks/kiwix_install.yml +++ b/roles/kiwix/tasks/kiwix_install.yml @@ -170,5 +170,5 @@ value: "{{ iiab_zim_path }}" - option: kiwix_library_xml value: "{{ kiwix_library_xml }}" - - option: enabled + - option: kiwix_enabled value: "{{ kiwix_enabled }}" diff --git a/roles/kiwix/templates/iiab-make-kiwix-lib.py b/roles/kiwix/templates/iiab-make-kiwix-lib.py old mode 100644 new mode 100755 index 3394f510a..b966d6240 --- a/roles/kiwix/templates/iiab-make-kiwix-lib.py +++ b/roles/kiwix/templates/iiab-make-kiwix-lib.py @@ -22,16 +22,18 @@ import shlex import ConfigParser import xml.etree.ElementTree as ET import argparse +import fnmatch IIAB_PATH='/etc/iiab' if not IIAB_PATH in sys.path: sys.path.append(IIAB_PATH) from iiab_env import get_iiab_env +KIWIX_CAT = IIAB_PATH + '/kiwix_catalog.json' # Config Files # iiab_ini_file should be in {{ iiab_env_file }} (/etc/iiab/iiab.env) ? -iiab_ini_file = "{{ iiab_ini_file }}" # nominally /etc/iiab/iiab.ini -#iiab_ini_file = "/etc/iiab/iiab.ini" # comment out after testing +#iiab_ini_file = "{{ iiab_ini_file }}" # nominally /etc/iiab/iiab.ini +iiab_ini_file = "/etc/iiab/iiab.ini" # comment out after testing IIAB_INI = get_iiab_env('IIAB_INI') # future if IIAB_INI: @@ -46,6 +48,10 @@ kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage" doc_root = get_iiab_env('WWWROOT') zim_version_idx_dir = doc_root + "/common/assets/" zim_version_idx_file = "zim_version_idx.json" +#zim_version_idx_file = "zim_version_idx_test.json" +menuDefs = doc_root + "/js-menu/menu-files/menu-defs/" +menuImages = doc_root + "/js-menu/menu-files/images/" +menuJsonPath = doc_root + "/home/menu.json" old_zim_map = {"bad.zim" : "unparseable name"} @@ -95,12 +101,8 @@ def main(): if item not in path_to_id_map: add_libr_xml(kiwix_library_xml, zim_path, item, zim_files[item]) - # Write Version Map - if os.path.isdir(zim_version_idx_dir): - with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp: - json.dump(zim_versions, fp) - else: - print zim_version_idx_dir + " not found." + print("Writing zim_versions_idx") + write_zim_versions_idx() sys.exit() def get_zim_list(path): @@ -113,6 +115,7 @@ def get_zim_list(path): for filename in flist: zimpos = filename.find(".zim") if zimpos != -1: + zim_info = {} filename = filename[:zimpos] zimname = "content/" + filename + ".zim" zimidx = "index/" + filename + ".zim.idx" @@ -130,7 +133,8 @@ def get_zim_list(path): if filename.rfind("-") < 0: # non-canonical name ulpos = filename[:ulpos].rfind("_") wiki_name = filename[:ulpos] - zim_versions[wiki_name] = filename # if there are multiples, last should win + zim_info['file_name'] = filename + zim_versions[wiki_name] = zim_info # if there are multiples, last should win return files_processed def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv @@ -159,7 +163,7 @@ def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from i def rem_libr_xml(id): command = kiwix_manage + " " + kiwix_library_xml + " remove " + id - print command + #print command args = shlex.split(command) try: outp = subprocess.check_output(args) @@ -171,13 +175,13 @@ def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx): command = kiwix_manage + " " + kiwix_library_xml + " add " + zim_path + "/" + zimname if zimidx: command += " -i " + zim_path + "/" + zimidx - print command + #print command args = shlex.split(command) try: outp = subprocess.check_output(args) except: #skip things that don't work - print 'skipping ' + zimname + #print 'skipping ' + zimname pass def init(): @@ -202,8 +206,102 @@ def parse_args(): parser.add_argument("-v", "--verbose", help="Print messages.", action="store_true") return parser.parse_args() -# Now start the application +def write_zim_versions_idx(): + global zim_versions + zims_installed,path_to_id_map = read_library_xml(kiwix_library_xml) + for perma_ref in zim_versions: + zim_versions[perma_ref]['menu_item'] = find_menuitem_from_zimname(perma_ref) + articlecount,mediacount,size,tags,lang,date = \ + get_substitution_data(perma_ref, zims_installed, path_to_id_map) + zim_versions[perma_ref]['article_count'] = articlecount + zim_versions[perma_ref]['media_count'] = mediacount + size = human_readable(size) + zim_versions[perma_ref]['size'] = size + zim_versions[perma_ref]['tags'] = tags + zim_versions[perma_ref]['language'] = lang + zim_versions[perma_ref]['zim_date'] = date + # Write Version Map + if os.path.isdir(zim_version_idx_dir): + with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp: + fp.write(json.dumps(zim_versions,indent=2 )) + fp.close() + else: + print zim_version_idx_dir + " not found." + +def get_substitution_data(perma_ref,zims_installed, path_to_id_map): + #reconstruct the path in the id map + path = 'content/' + zim_versions[perma_ref]['file_name'] + '.zim' + id = path_to_id_map[path] + item = zims_installed[id] + if len(item) != 0 or perma_ref == 'test': + mediacount = item.get('mediaCount','') + articlecount = item.get('articleCount','') + size = item.get('size','') + tags = item.get('tags','') + lang = item.get('language','') + if len(lang) > 2: + lang = lang[:2] + date = item.get('date','') + return (articlecount,mediacount,size,tags,lang,date) + return ('0','0','0','0','0','0') + +def get_menu_def_zimnames(intended_use='zim'): + menu_def_dict = {} + os.chdir(menuDefs) + for filename in os.listdir('.'): + if fnmatch.fnmatch(filename, '*.json'): + try: + with open(filename,'r') as json_file: + readstr = json_file.read() + data = json.loads(readstr) + except: + print("failed to parse %s"%filename) + print(readstr) + if data.get('intended_use','') != 'zim': + continue + zimname = data.get('zim_name','') + if zimname != '': + menu_def_dict[data['zim_name']] = menuDefs + filename + return menu_def_dict + +def find_menuitem_from_zimname(zimname): + defs = get_menu_def_zimnames() + defs_filename = defs.get(zimname,'') + if defs_filename != '': + #print("reading menu-def:%s"%defs_filename) + with open(defs_filename,'r') as json_file: + readstr = json_file.read() + data = json.loads(readstr) + return data.get('menu_item_name','') + return '' + +def get_kiwix_catalog_item(perma_ref): + # Read the kiwix catalog + with open(KIWIX_CAT, 'r') as kiwix_cat: + json_data = kiwix_cat.read() + download = json.loads(json_data) + zims = download['zims'] + for uuid in zims.keys(): + #print("%s %s"%(zims[uuid]['perma_ref'],perma_ref,)) + if zims[uuid]['perma_ref'] == perma_ref: + return zims[uuid] + return {} + +def human_readable(num): + # return 3 significant digits and unit specifier + num = float(num) + units = [ '','K','M','G'] + for i in range(4): + if num<10.0: + return "%.2f%s"%(num,units[i]) + if num<100.0: + return "%.1f%s"%(num,units[i]) + if num < 1000.0: + return "%.0f%s"%(num,units[i]) + num /= 1000.0 + +# Now start the application if __name__ == "__main__": # Run the main routine diff --git a/roles/kolibri/tasks/main.yml b/roles/kolibri/tasks/main.yml index 961ee1bb2..57b87750d 100644 --- a/roles/kolibri/tasks/main.yml +++ b/roles/kolibri/tasks/main.yml @@ -97,5 +97,5 @@ value: "{{ kolibri_exec_path }}" - option: kolibri_port value: "{{ kolibri_http_port }}" - - option: enabled + - option: kolibri_enabled value: "{{ kolibri_enabled }}" diff --git a/roles/network/templates/gateway/iiab-gen-iptables b/roles/network/templates/gateway/iiab-gen-iptables index cdc87f326..94f2f49cc 100755 --- a/roles/network/templates/gateway/iiab-gen-iptables +++ b/roles/network/templates/gateway/iiab-gen-iptables @@ -58,6 +58,7 @@ calibre_port={{ calibre_port }} kiwix_port={{ kiwix_port }} kalite_server_port={{ kalite_server_port }} kolibri_http_port={{ kolibri_http_port }} +cups_port={{ cups_port }} transmission_http_port={{ transmission_http_port }} transmission_peer_port={{ transmission_peer_port }} sugarizer_port={{ sugarizer_port }} @@ -93,6 +94,7 @@ if [ "$services_externally_visible" == "True" ]; then $IPTABLES -A INPUT -p tcp --dport $kalite_server_port -m state --state NEW -i $wan -j ACCEPT $IPTABLES -A INPUT -p tcp --dport $kolibri_http_port -m state --state NEW -i $wan -j ACCEPT $IPTABLES -A INPUT -p tcp --dport $calibre_port -m state --state NEW -i $wan -j ACCEPT + $IPTABLES -A INPUT -p tcp --dport $cups_port -m state --state NEW -i $wan -j ACCEPT $IPTABLES -A INPUT -p tcp --dport $sugarizer_port -m state --state NEW -i $wan -j ACCEPT $IPTABLES -A INPUT -p tcp --dport $transmission_http_port -m state --state NEW -i $wan -j ACCEPT $IPTABLES -A INPUT -p tcp --dport $transmission_peer_port -m state --state NEW -i $wan -j ACCEPT diff --git a/roles/sugarizer/tasks/main.yml b/roles/sugarizer/tasks/main.yml index 33def01e3..ef494b463 100644 --- a/roles/sugarizer/tasks/main.yml +++ b/roles/sugarizer/tasks/main.yml @@ -255,5 +255,5 @@ value: Sugarizer - option: description value: '"The Sugar Learning Platform began with the famous One Laptop Per Child project, written in Python. Sugarizer is the new HTML/JavaScript implementation of Sugar, usable in most all browsers."' - - option: enabled + - option: sugarizer_enabled value: "{{ sugarizer_enabled }}" diff --git a/roles/teamviewer/tasks/main.yml b/roles/teamviewer/tasks/main.yml index 2162d4efa..e8dfce7a6 100644 --- a/roles/teamviewer/tasks/main.yml +++ b/roles/teamviewer/tasks/main.yml @@ -19,7 +19,7 @@ value: teamviewer - option: description value: '"TeamViewer - the All-In-One Software for Remote Support and Online Meetings"' - - option: installed + - option: teamviewer_install value: "{{ teamviewer_install }}" - - option: enabled + - option: teamviewer_enabled value: "{{ teamviewer_enabled }}" diff --git a/vars/default_vars.yml b/vars/default_vars.yml index fbb14cdd8..8ea6396f6 100644 --- a/vars/default_vars.yml +++ b/vars/default_vars.yml @@ -226,6 +226,7 @@ postgresql_enabled: False # Common UNIX Printing System (CUPS) cups_install: False cups_enabled: False +cups_port: 631 # Samba. Take a security audit seriously before deploying this. samba_install: False From 6aafc58984649e57c4a1b91358478eae90917065 Mon Sep 17 00:00:00 2001 From: A Holt Date: Mon, 14 Jan 2019 08:57:47 -0500 Subject: [PATCH 2/3] Update default_vars.yml --- vars/default_vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/default_vars.yml b/vars/default_vars.yml index a58d8a361..3a9404344 100644 --- a/vars/default_vars.yml +++ b/vars/default_vars.yml @@ -486,7 +486,7 @@ calibreweb_home: "{{ content_base }}/calibre-web" # /library/calibre-web # Platforms - turn all off and let /opt/iiab/iiab/vars/.yml turn on as appropriate -# Wide to narrow +# Wide to narrow (insofar as poss) is_debuntu: False is_ubuntu: False is_ubuntu_18: False @@ -496,9 +496,9 @@ is_debian: False is_debian_10: False is_debian_9: False is_debian_8: False -is_rpi: False is_raspbian_9: False is_raspbian_8: False +is_rpi: False is_redhat: False is_centos: False From 3966787ac95648bc1fe404cbef66dcdc2f4e93d1 Mon Sep 17 00:00:00 2001 From: A Holt Date: Mon, 14 Jan 2019 09:06:39 -0500 Subject: [PATCH 3/3] Update packages.yml --- roles/2-common/tasks/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/2-common/tasks/packages.yml b/roles/2-common/tasks/packages.yml index e27f24f75..fbb938ea1 100644 --- a/roles/2-common/tasks/packages.yml +++ b/roles/2-common/tasks/packages.yml @@ -18,7 +18,7 @@ apt: deb: "{{ iiab_download_url }}/usbmount_0.0.14.1_all.deb" #timeout: "{{ download_timeout }}" # Ansible's apt module doesn't support timeout parameter; that's ok as usbmount_0.0.14.1_all.deb is only 10KB - when: internet_available and is_debian_9 + when: internet_available and is_debian_9 and not is_raspbian_9 - name: "Install 7 deb/apt packages: avahi-daemon, avahi-discover, exfat-fuse, exfat-utils, inetutils-syslogd, libnss-mdns, wpasupplicant (debuntu)" package: