move pylib.yml to own role

roles/pylibs/tasks/main.yml

@@ -0,0 +1,11 @@
+- name: Install iiab lib files
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - { src: 'iiab_const.py.j2', dest: '{{ py3_dist_path }}/iiab/iiab_const.py' }
+    - { src: 'iiab_lib.py', dest: '{{ py3_dist_path }}/iiab/iiab_lib.py' }
+    - { src: 'iiab_env.py.j2', dest: '{{ iiab_etc_path }}/iiab_env.py' }

roles/pylibs/templates/iiab_const.py.j2

@@ -0,0 +1,21 @@
+# IIAB constants
+
+iiab_etc_path = '{{ iiab_etc_path }}'
+iiab_env_file = '{{ iiab_env_file }}'
+iiab_ini_file = '{{ iiab_ini_file }}'
+iiab_local_vars_file = '{{ iiab_local_vars_file }}'
+iiab_base = '{{ iiab_base }}'
+iiab_repo_dir = '{{ iiab_dir }}'
+iiab_state_file = '{{ iiab_state_file }}'
+
+kiwix_cat_path = iiab_etc_path + '/kiwix_catalog.json'
+zim_path = '{{ iiab_zim_path }}'
+kiwix_library_xml = zim_path + '/library.xml'
+iiab_base_path = '{{ iiab_base }}'
+kiwix_manage = iiab_base_path + '/kiwix/bin/kiwix-manage'
+
+doc_root = '{{ doc_root }}'
+assets_dir = doc_root + '/common/assets/'
+lang_codes_path = assets_dir + 'lang_codes.json'
+
+old_zim_map = {"bad.zim" : "unparseable name"}

roles/pylibs/templates/iiab_env.py.j2

@@ -0,0 +1,27 @@
+#!/usr/bin/python3
+# read iiab.env from python
+
+def get_iiab_env(name):
+   """ read iiab.env file for a value, return "" if does not exist"""
+   try:
+      fd = open("{{ iiab_env_file }}","r")
+      for line in fd:
+         line = line.lstrip()
+         line = line.rstrip('\n')
+         if len(line) == 0:
+            continue
+         if line[0] == "#":
+            continue
+         if line.find("=") == -1:
+            continue
+         chunks = line.split('=')
+         if chunks[0] == name:
+            return chunks[1]
+      return("")
+   except:
+      return("")
+   finally:
+      fd.close()
+
+if __name__ == "__main__":
+   print(get_iiab_env("WWWROOT"))

roles/pylibs/templates/iiab_lib.py

@@ -0,0 +1,209 @@
+'''
+Common functions for IIAB
+Admin Console functions are in adm_lib.py
+'''
+import os
+import json
+import subprocess
+import shlex
+import xml.etree.ElementTree as ET
+import iiab.iiab_const as CONST
+
+lang_codes = {}
+lang_iso2_codes = {}
+
+def get_zim_list(path):
+    '''
+    Get a list of installed zims in the passed path
+
+    Args:
+      path (str): The path to search
+
+    Returns:
+      files_processed (dict): A dict all zims found and any index directory (now obsolete)
+      zim_versions (dict): A dict that translates generic zim names to physically installed
+    '''
+
+    files_processed = {}
+    zim_versions = {} # we don't need this unless adm cons is installed, but easier to compute now
+    content = path + "/content/"
+    index = path + "/index/"
+    flist = os.listdir(content)
+    flist.sort()
+    for filename in flist:
+        zimpos = filename.find(".zim")
+        if zimpos != -1:
+            zim_info = {}
+            filename = filename[:zimpos]
+            zimname = "content/" + filename + ".zim"
+            zimidx = "index/" + filename + ".zim.idx"
+            if zimname not in files_processed:
+                if not os.path.isdir(path + "/" + zimidx): # only declare index if exists (could be embedded)
+                    zimidx = None
+                files_processed[zimname] = zimidx
+                zimname = content + filename + ".zim"
+                zimidx = index + filename + ".zim.idx"
+                if filename in CONST.old_zim_map: # handle old names that don't parse
+                    perma_ref = CONST.old_zim_map[filename]
+                else:
+                    ulpos = filename.rfind("_")
+                    # but old gutenberg and some other names are not canonical
+                    if filename.rfind("-") < 0: # non-canonical name
+                        ulpos = filename[:ulpos].rfind("_")
+                    perma_ref = filename[:ulpos]
+                zim_info['file_name'] = filename
+                zim_versions[perma_ref] = zim_info # if there are multiples, last should win
+    return files_processed, zim_versions
+
+def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv
+    '''
+    Read zim properties from library.xml
+    Returns dict of library.xml and map of zim id to zim file name (under <dev>/library/zims)
+
+    Args:
+      lib_xml_file (str): Path to file to read. Can be on removable device
+      kiwix_exclude_attr (list): Zim properties to exclude from return
+
+    Returns:
+      zims_installed (dict): A dictionary holding all installed zims and their attributes
+      path_to_id_map (dict): A dictionary that translates zim ids to physical names
+    '''
+
+    kiwix_exclude_attr.append("id") # don't include id
+    kiwix_exclude_attr.append("favicon") # don't include large favicon
+    zims_installed = {}
+    path_to_id_map = {}
+    try:
+        tree = ET.parse(lib_xml_file)
+        root = tree.getroot()
+        for child in root:
+            attributes = {}
+            if 'id' not in child.attrib: # is this necessary? implies there are records with no book id which would break index for removal
+                print("xml record missing Book Id")
+            zim_id = child.attrib['id']
+            for attr in child.attrib:
+                if attr not in kiwix_exclude_attr:
+                    attributes[attr] = child.attrib[attr] # copy if not id or in exclusion list
+            zims_installed[zim_id] = attributes
+            path_to_id_map[child.attrib['path']] = zim_id
+    except IOError:
+        zims_installed = {}
+    return zims_installed, path_to_id_map
+
+def rem_libr_xml(zim_id, kiwix_library_xml):
+    '''
+    Remove a zim from library.xml
+
+    Args:
+      zim_id (uuid): Id of the zim to remove
+      lib_xml_file (str): Path to file to read. Can be on removable device
+    '''
+
+    command = CONST.kiwix_manage + " " + kiwix_library_xml + " remove " + zim_id
+    #print command
+    args = shlex.split(command)
+    try:
+        outp = subprocess.check_output(args)
+    except subprocess.CalledProcessError as e:
+        if e.returncode != 2: # skip bogus file open error in kiwix-manage
+            print(outp)
+
+def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx):
+    '''
+    Add a zim to library.xml
+
+    Args:
+      kiwix_library_xml (str): Name (path) of library.xml file
+      zim_path (str): Path to zim file to add
+      zimname (str): Name of zim file to add
+      zimidx (str): Path to separate idx directory (obsolete)
+
+    '''
+    command = CONST.kiwix_manage + " " + kiwix_library_xml + " add " + zim_path + "/" + zimname
+    if zimidx:
+        command += " -i " + zim_path + "/" + zimidx
+    #print command
+    args = shlex.split(command)
+    try:
+        outp = subprocess.check_output(args)
+
+    except: #skip things that don't work
+        #print 'skipping ' + zimname
+        pass
+
+def read_lang_codes():
+    '''Populate the global lang_codes dictionary from CONST.lang_codes_path json file'''
+
+    global lang_codes
+    with open(CONST.lang_codes_path, "r") as f:
+        reads = f.read()
+        #print("menu.json:%s"%reads)
+        lang_codes = json.loads(reads)
+
+    # create iso2 index
+    for lang in lang_codes:
+        lang_iso2_codes[lang_codes[lang]['iso2']  ] = lang
+
+# there is a different algorithm in get_zim_list above
+def calc_perma_ref(uri):
+    '''Given a path or url return the generic zim name'''
+    url_slash = uri.split('/')
+    url_end = url_slash[-1] # last element
+    file_ref = url_end.split('.zim')[0] # true for both internal and external index
+    perma_ref_parts = file_ref.split('_')
+    perma_ref = perma_ref_parts[0]
+    if len(perma_ref_parts) > 1:
+        perma_ref_parts = perma_ref_parts[0:len(perma_ref_parts) - 1] # all but last, which should be date
+        for part in perma_ref_parts[1:]: # start with 2nd
+            if not part.isdigit():
+                perma_ref += "_" + part
+    return perma_ref
+
+def kiwix_lang_to_iso2(zim_lang_code):
+    '''Lookup the iso2 equivalent of a zim language code'''
+    return lang_codes[zim_lang_code]['iso2']
+
+def human_readable(num):
+    '''Convert a number to a human readable string'''
+    # return 3 significant digits and unit specifier
+    # TFM 7/15/2019 change to factor of 1024, not 1000 to match similar calcs elsewhere
+    num = float(num)
+    units = ['', 'K', 'M', 'G']
+    for i in range(4):
+        if num < 10.0:
+            return "%.2f%s"%(num, units[i])
+        if num < 100.0:
+            return "%.1f%s"%(num, units[i])
+        if num < 1000.0:
+            return "%.0f%s"%(num, units[i])
+        num /= 1024.0
+
+# Environment Functions
+
+def get_iiab_env(name):
+    ''' read iiab.env file for a value, return "" if does not exist. return all value for *'''
+    iiab_env = {}
+    iiab_env_var = ''
+    try:
+        fd = open("/etc/iiab/iiab.env", "r")
+        for line in fd:
+            line = line.lstrip()
+            line = line.rstrip('\n')
+            if len(line) == 0:
+                continue
+            if line[0] == "#":
+                continue
+            if line.find("=") == -1:
+                continue
+            chunks = line.split('=')
+            iiab_env[chunks[0]] = chunks[1]
+            if chunks[0] == name:
+                iiab_env_var = chunks[1]
+    except:
+        pass
+    finally:
+        fd.close()
+    if name == '*':
+        return iiab_env
+    else:
+        return iiab_env_var