Merge pull request #318 from iiab/master

sync from iiab:master

roles/2-common/tasks/fl.yml

@@ -1,6 +1,6 @@
 # fl.yml signifies "file layout"
 
-- name: Create 19 directories with ownership root:root and permissions 0755 (1 in /etc, 3 in {{ iiab_base }} and 15 in /library)    # iiab_base is /opt/iiab
+- name: Create directories with ownership root:root and permissions 0755 (1 in /etc, 3 in {{ iiab_base }} and 15 in /library)    # iiab_base is /opt/iiab
   file:
     path: "{{ item }}"
     owner: root
@@ -29,6 +29,7 @@
     - "{{ doc_root }}/common/images"
     - "{{ doc_root }}/common/assets"
     - "{{ doc_root }}/common/services"
+    - "{{ py3_dist_path }}/iiab"
 
 - name: Symlink from {{ doc_root }}/common/webfonts to {{ doc_root }}/common/fonts
   file:

roles/2-common/tasks/main.yml

@@ -6,6 +6,8 @@
 - name: Create IIAB directory structure ("file layout")
   include_tasks: fl.yml
 
+- include_tasks: pylib.yml
+
 - include_tasks: centos.yml
   when: ansible_distribution == "CentOS"
 

roles/2-common/tasks/pylib.yml

@@ -0,0 +1,10 @@
+- name: Install iiab lib files
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - { src: 'iiab_const.py.j2', dest: '{{ py3_dist_path }}/iiab/iiab_const.py' }
+    - { src: 'iiab_lib.py', dest: '{{ py3_dist_path }}/iiab/iiab_lib.py' }

roles/2-common/templates/iiab_const.py.j2

@@ -0,0 +1,17 @@
+# IIAB constants
+
+iiab_etc_path = '{{ iiab_etc_path }}'
+iiab_env_file = '{{ iiab_env_file }}'
+iiab_ini_file = '{{ iiab_ini_file }}'
+
+kiwix_cat_path = iiab_etc_path + '/kiwix_catalog.json'
+zim_path = '{{ iiab_zim_path }}'
+kiwix_library_xml = zim_path + '/library.xml'
+iiab_base_path = '{{ iiab_base }}'
+kiwix_manage = iiab_base_path + '/kiwix/bin/kiwix-manage'
+
+doc_root = '{{ doc_root }}'
+assets_dir = doc_root + '/common/assets/'
+lang_codes_path = assets_dir + 'lang_codes.json'
+
+old_zim_map = {"bad.zim" : "unparseable name"}

roles/2-common/templates/iiab_lib.py

@@ -0,0 +1,170 @@
+# iiab_lib.py
+# common functions for IIAB
+# Admin Console functions are in adm_lib.py
+
+import os, sys, syslog
+import pwd, grp
+import time
+from datetime import date, datetime
+import json
+import yaml
+import re
+import subprocess
+import shlex
+import configparser
+import xml.etree.ElementTree as ET
+import argparse
+import iiab.iiab_const as cons
+
+lang_codes = {}
+
+def get_zim_list(path):
+    files_processed = {}
+    zim_versions = {} # we don't need this unless adm cons is installed, but easier to compute now
+    zim_list = []
+    content = path + "/content/"
+    index = path + "/index/"
+    flist = os.listdir(content)
+    flist.sort()
+    for filename in flist:
+        zimpos = filename.find(".zim")
+        if zimpos != -1:
+            zim_info = {}
+            filename = filename[:zimpos]
+            zimname = "content/" + filename + ".zim"
+            zimidx = "index/" + filename + ".zim.idx"
+            if zimname not in files_processed:
+                if not os.path.isdir (path + "/" + zimidx): # only declare index if exists (could be embedded)
+                    zimidx = None
+                files_processed[zimname] = zimidx
+                zimname = content + filename + ".zim"
+                zimidx = index + filename + ".zim.idx"
+                if filename in cons.old_zim_map: # handle old names that don't parse
+                    perma_ref = cons.old_zim_map[filename]
+                else:
+                    ulpos = filename.rfind("_")
+                    # but old gutenberg and some other names are not canonical
+                    if filename.rfind("-") < 0: # non-canonical name
+                        ulpos = filename[:ulpos].rfind("_")
+                    perma_ref = filename[:ulpos]
+                zim_info['file_name'] = filename
+                zim_versions[perma_ref] = zim_info # if there are multiples, last should win
+    return files_processed, zim_versions
+
+def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv
+    # returns dict of library.xml and map of zim id to zim file name (under <dev>/library/zims)
+
+    kiwix_exclude_attr.append("id") # don't include id
+    kiwix_exclude_attr.append("favicon") # don't include large favicon
+    zims_installed = {}
+    path_to_id_map = {}
+    try:
+        tree = ET.parse(lib_xml_file)
+        root = tree.getroot()
+        xml_item_no = 0
+        for child in root:
+            #xml_item_no += 1 # hopefully this is the array number
+            attributes = {}
+            if 'id' not in child.attrib: # is this necessary? implies there are records with no book id which would break index for removal
+                print("xml record missing Book Id")
+            id = child.attrib['id']
+            for attr in child.attrib:
+                if attr not in kiwix_exclude_attr:
+                    attributes[attr] = child.attrib[attr] # copy if not id or in exclusion list
+            zims_installed[id] = attributes
+            path_to_id_map[child.attrib['path']] = id
+    except IOError:
+        zims_installed = {}
+    return zims_installed, path_to_id_map
+
+def rem_libr_xml(id):
+    command = cons.kiwix_manage + " " + kiwix_library_xml + " remove " + id
+    #print command
+    args = shlex.split(command)
+    try:
+        outp = subprocess.check_output(args)
+    except subprocess.CalledProcessError as e:
+        if e.returncode != 2: # skip bogus file open error in kiwix-manage
+            print(outp)
+
+def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx):
+    command = cons.kiwix_manage + " " + kiwix_library_xml + " add " + cons.zim_path + "/" + zimname
+    if zimidx:
+        command += " -i " + zim_path + "/" + zimidx
+    #print command
+    args = shlex.split(command)
+    try:
+        outp = subprocess.check_output(args)
+
+    except: #skip things that don't work
+        #print 'skipping ' + zimname
+        pass
+
+def read_lang_codes():
+    global lang_codes
+    with open(cons.lang_codes_path,"r") as f:
+        reads = f.read()
+        #print("menu.json:%s"%reads)
+        lang_codes = json.loads(reads)
+
+# there is a different algorithm in get_zim_list above
+
+def calc_perma_ref(uri):
+    url_slash = uri.split('/')
+    url_end = url_slash[-1] # last element
+    file_ref = url_end.split('.zim')[0] # true for both internal and external index
+    perma_ref_parts = file_ref.split('_')
+    perma_ref = perma_ref_parts[0]
+    if len(perma_ref_parts) > 1:
+        perma_ref_parts = perma_ref_parts[0:len(perma_ref_parts) - 1] # all but last, which should be date
+        for part in perma_ref_parts[1:]: # start with 2nd
+            if not part.isdigit():
+                perma_ref += "_" + part
+    return perma_ref
+
+def kiwix_lang_to_iso2(zim_lang_code):
+    return lang_codes[zim_lang_code]['iso2']
+
+def human_readable(num):
+    # return 3 significant digits and unit specifier
+    # TFM 7/15/2019 change to factor of 1024, not 1000 to match similar calcs elsewhere
+    num = float(num)
+    units = [ '','K','M','G']
+    for i in range(4):
+        if num<10.0:
+            return "%.2f%s"%(num,units[i])
+        if num<100.0:
+            return "%.1f%s"%(num,units[i])
+        if num < 1000.0:
+            return "%.0f%s"%(num,units[i])
+        num /= 1024.0
+
+# Environment Functions
+
+def get_iiab_env(name):
+    """ read iiab.env file for a value, return "" if does not exist. return all value for *"""
+    iiab_env = {}
+    iiab_env_var = ''
+    try:
+        fd = open(cons.iiab_env_file,"r")
+        for line in fd:
+            line = line.lstrip()
+            line = line.rstrip('\n')
+            if len(line) == 0:
+                continue
+            if line[0] == "#":
+                continue
+            if line.find("=") == -1:
+                continue
+            chunks = line.split('=')
+            iiab_env[chunks[0]] = chunks[1]
+            if chunks[0] == name:
+                iiab_env_var = chunks[1]
+    except:
+        pass
+    finally:
+        fd.close()
+    if name == '*':
+        return iiab_env
+    else:
+        return iiab_env_var

scripts/iiab-diagnostics

@@ -147,6 +147,7 @@ cat_file /.iiab-image
 cat_file /etc/iiab/iiab.env
 cat_file /etc/iiab/iiab.ini
 cat_file /etc/iiab/local_vars.yml
+cat_file /etc/iiab/iiab_state.yml
 cat_file /etc/iiab/config_vars.yml
 cat_file /etc/resolv.conf
 cat_file /etc/network/interfaces

scripts/iiab-diagnostics.README.md

@@ -60,4 +60,4 @@ But first off, the file is compiled by harvesting 1 + 6 kinds of things:
 
 ## Source Code
 
-Please look over the bottom of [iiab-diagnostics](iiab-diagnostics) (lines 100-197 especially) to learn more about which common IIAB files and commands make this rapid troubleshooting possible.
+Please look over the bottom of [iiab-diagnostics](iiab-diagnostics) (lines 100-198 especially) to learn more about which common IIAB files and commands make this rapid troubleshooting possible.

vars/default_vars.yml

@@ -8,12 +8,14 @@
 iiab_base_ver: 7.1
 iiab_revision: 0
 
+iiab_etc_path: /etc/iiab
+
 # Main configuration file
-iiab_local_vars_file: /etc/iiab/local_vars.yml
+iiab_local_vars_file: "{{ iiab_etc_path }}/local_vars.yml"
 
 # Installation status files
-iiab_env_file: /etc/iiab/iiab.env
-iiab_ini_file: /etc/iiab/iiab.ini
+iiab_env_file: "{{ iiab_etc_path }}/iiab.env"
+iiab_ini_file: "{{ iiab_etc_path }}/iiab.ini"
 
 iiab_base: /opt/iiab
 iiab_dir: "{{ iiab_base }}/iiab"
@@ -26,6 +28,9 @@ content_base: "/library"
 doc_base: "{{ content_base }}/www"
 doc_root: "{{ doc_base }}/html"
 
+# Python 3 dist path
+py3_dist_path: /usr/lib/python3/dist-packages
+
 # Ansible's default timeout for "get_url:" downloads (10 seconds) often fails
 download_timeout: 200
 
@@ -305,8 +310,8 @@ azuracast_http_port: 10080
 azuracast_https_port: 10443
 
 # AzuraCast needs many ports in the 8000:8100 range by default, but IIAB services
-# conflict with those ports so this variable below sets a sane prefix. 
-# e.g. setting the below variable to 10 will result in port ranges 10000-10100 
+# conflict with those ports so this variable below sets a sane prefix.
+# e.g. setting the below variable to 10 will result in port ranges 10000-10100
 # being reserved for AzuraCast:
 azuracast_port_range_prefix: 10