0.3 patch1 (#1377)

* expand zim_versions_idx to include menuItem name, mediaCount, articleCount, size

* create a stub menuItem if none exists

* remember to change consumer of zim_version_idx in iiab-admin

* comment out some debugging prints

* do not change the name of a variable in iiab.ini

* consistent variable names for *_enabled

* start work on menus for enabled iiab roles

* changes to display cups in home menu

* remove reference to admin console which may not be installed

* print error message

* dict.get errors

* handle undefined menuItem"

* handle undefined menuItem again

* some work on logos

* break out the zim_versions_idx routines for use by update_menus in admin-console

* remove old menuDef creation code

* new variable names for zim_versions_idx

* missing the tags data in zim_versions_idx

* un-break-apart iiab-make-kiwix-lib.py

* return an empty string

* make size human_readable

* getting console and iiab to work together on menus

* add the new zim_date field to zim_versions_idx

* get the latest into repo

* found lost code iiab-make-kiwix-lib.py

* Create Lokole admin user during setup

* Add requested content to lokole readme
Per iiab/iiab#1293

* Update default password

* Add Lokole-IIAB user manual

* Update default_vars.yml

* Update local_vars_min.yml

* Update local_vars_min.yml

* Update local_vars_medium.yml

* Update local_vars_min.yml

* Update default_vars.yml

* Update local_vars_big.yml

* Update local_vars_medium.yml

* Update default_vars.yml

* Update default_vars.yml

* Update local_vars_min.yml

* Update local_vars_medium.yml

* Update local_vars_big.yml

* Update default_vars.yml

* Update default_vars.yml

* Update local_vars_big.yml

* Update local_vars_medium.yml

* Update local_vars_min.yml

* Update default_vars.yml

* Update local_vars_big.yml

* Update local_vars_medium.yml

* Update local_vars_min.yml

* Change admin username to uppercase

* Revert "Lokole: change admin to Admin per IIAB app norms"

* Update main.yml

* Update README.rst

* Update capture-wsgi.py

* Update main.yml

* Update main.yml

* Update default_vars.yml

* Update local_vars_big.yml

* Update local_vars_medium.yml

* Update local_vars_min.yml

* Update main.yml

* Update main.yml

* Update main.yml

* Update main.yml

* Update default_vars.yml

* Update local_vars_big.yml

* Update local_vars_medium.yml

* Update local_vars_min.yml

* Address TODOs in Lokole documentation

See https://github.com/ascoderu/opwen-webapp/issues/81

* Update main.yml

* Update local_vars_big.yml

* Update local_vars_medium.yml

* Update local_vars_big.yml

* Update local_vars_min.yml

* Update default_vars.yml

* expand zim_versions_idx to include menuItem name, mediaCount, articleCount, size

* create a stub menuItem if none exists

* remember to change consumer of zim_version_idx in iiab-admin

* comment out some debugging prints

* do not change the name of a variable in iiab.ini

* consistent variable names for *_enabled

* start work on menus for enabled iiab roles

* changes to display cups in home menu

* remove reference to admin console which may not be installed

* print error message

* dict.get errors

* handle undefined menuItem"

* handle undefined menuItem again

* some work on logos

* break out the zim_versions_idx routines for use by update_menus in admin-console

* remove old menuDef creation code

* new variable names for zim_versions_idx

* missing the tags data in zim_versions_idx

* un-break-apart iiab-make-kiwix-lib.py

* return an empty string

* make size human_readable

* getting console and iiab to work together on menus

* add the new zim_date field to zim_versions_idx

* get the latest into repo

* found lost code iiab-make-kiwix-lib.py

roles/calibre/tasks/main.yml

@@ -147,5 +147,5 @@
     value: "{{ calibre_dbpath }}"
   - option: port
     value: "{{ calibre_port }}"
-  - option: enabled
+  - option: calibre_enabled
     value: "{{ calibre_enabled }}"

roles/cups/tasks/main.yml

@@ -75,5 +75,5 @@
       value: '"CUPS (Common UNIX Printing System) is a modular printing system that allows a computer to act as a print server. A computer running CUPS is a host that can accept print jobs from client computers, process them, and send them to the appropriate printer."'
     - option: installed
       value: "{{ cups_install }}"
-    - option: enabled
+    - option: cups_enabled
       value: "{{ cups_enabled }}"

roles/elgg/tasks/main.yml

@@ -161,5 +161,5 @@
       value: '"Elgg is an award-winning social networking engine, delivering the building blocks that enable businesses, schools, universities and associations to create their own fully-featured social networks and applications."'
     - option: path
       value: /opt/elgg
-    - option: enabled
+    - option: elgg_enabled
       value: "{{ elgg_enabled }}"

roles/kalite/tasks/main.yml

@@ -51,7 +51,7 @@
     value: "{{ kalite_root }}"
   - option: port
     value: "{{ kalite_server_port }}"
-  - option: enabled
+  - option: kalite_enabled
     value: "{{ kalite_enabled }}"
   - option: cron_enabled
     value: "{{ kalite_cron_enabled }}"

roles/kiwix/tasks/kiwix_install.yml

@@ -170,5 +170,5 @@
     value: "{{ iiab_zim_path }}"
   - option: kiwix_library_xml
     value: "{{ kiwix_library_xml }}"
-  - option: enabled
+  - option: kiwix_enabled
     value: "{{ kiwix_enabled }}"

roles/kiwix/templates/iiab-make-kiwix-lib.py

@@ -22,16 +22,18 @@ import shlex
 import ConfigParser
 import xml.etree.ElementTree as ET
 import argparse
+import fnmatch
 
 IIAB_PATH='/etc/iiab'
 if not IIAB_PATH in sys.path:
     sys.path.append(IIAB_PATH)
 from iiab_env import get_iiab_env
+KIWIX_CAT = IIAB_PATH + '/kiwix_catalog.json'
 
 # Config Files
 # iiab_ini_file should be in {{ iiab_env_file }} (/etc/iiab/iiab.env) ?
-iiab_ini_file = "{{ iiab_ini_file }}" # nominally /etc/iiab/iiab.ini
-#iiab_ini_file = "/etc/iiab/iiab.ini" # comment out after testing
+#iiab_ini_file = "{{ iiab_ini_file }}" # nominally /etc/iiab/iiab.ini
+iiab_ini_file = "/etc/iiab/iiab.ini" # comment out after testing
 
 IIAB_INI = get_iiab_env('IIAB_INI') # future
 if IIAB_INI:
@@ -46,6 +48,10 @@ kiwix_manage = iiab_base_path + "/kiwix/bin/kiwix-manage"
 doc_root = get_iiab_env('WWWROOT')
 zim_version_idx_dir = doc_root + "/common/assets/"
 zim_version_idx_file = "zim_version_idx.json"
+#zim_version_idx_file = "zim_version_idx_test.json"
+menuDefs = doc_root + "/js-menu/menu-files/menu-defs/"
+menuImages = doc_root + "/js-menu/menu-files/images/"
+menuJsonPath = doc_root + "/home/menu.json"
 
 old_zim_map = {"bad.zim" : "unparseable name"}
 
@@ -95,12 +101,8 @@ def main():
           if item not in path_to_id_map:
               add_libr_xml(kiwix_library_xml, zim_path, item, zim_files[item])
 
-    # Write Version Map
-    if os.path.isdir(zim_version_idx_dir):
-        with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp:
-            json.dump(zim_versions, fp)
-    else:
-        print zim_version_idx_dir + " not found."
+    print("Writing zim_versions_idx")
+    write_zim_versions_idx()
     sys.exit()
 
 def get_zim_list(path):
@@ -113,6 +115,7 @@ def get_zim_list(path):
     for filename in flist:
         zimpos = filename.find(".zim")
         if zimpos != -1:
+            zim_info = {}
             filename = filename[:zimpos]
             zimname = "content/" + filename + ".zim"
             zimidx = "index/" + filename + ".zim.idx"
@@ -130,7 +133,8 @@ def get_zim_list(path):
                     if filename.rfind("-") < 0: # non-canonical name
                         ulpos = filename[:ulpos].rfind("_")
                     wiki_name = filename[:ulpos]
-                zim_versions[wiki_name] = filename # if there are multiples, last should win
+                zim_info['file_name'] = filename
+                zim_versions[wiki_name] = zim_info # if there are multiples, last should win
     return files_processed
 
 def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from iiab-cmdsrv
@@ -159,7 +163,7 @@ def read_library_xml(lib_xml_file, kiwix_exclude_attr=[""]): # duplicated from i
 
 def rem_libr_xml(id):
     command = kiwix_manage + " " + kiwix_library_xml + " remove " + id
-    print command
+    #print command
     args = shlex.split(command)
     try:
         outp = subprocess.check_output(args)
@@ -171,13 +175,13 @@ def add_libr_xml(kiwix_library_xml, zim_path, zimname, zimidx):
     command = kiwix_manage + " " + kiwix_library_xml + " add " + zim_path + "/" + zimname
     if zimidx:
           command += " -i " + zim_path + "/" + zimidx
-    print command
+    #print command
     args = shlex.split(command)
     try:
         outp = subprocess.check_output(args)
 
     except: #skip things that don't work
-        print 'skipping ' + zimname
+        #print 'skipping ' + zimname
         pass
 
 def init():
@@ -202,8 +206,102 @@ def parse_args():
     parser.add_argument("-v", "--verbose", help="Print messages.", action="store_true")
     return parser.parse_args()
 
-# Now start the application
+def write_zim_versions_idx():
+   global zim_versions
+   zims_installed,path_to_id_map = read_library_xml(kiwix_library_xml)
+   for perma_ref in zim_versions:
+      zim_versions[perma_ref]['menu_item'] = find_menuitem_from_zimname(perma_ref)
+      articlecount,mediacount,size,tags,lang,date = \
+           get_substitution_data(perma_ref, zims_installed, path_to_id_map)
+      zim_versions[perma_ref]['article_count'] = articlecount
+      zim_versions[perma_ref]['media_count'] = mediacount
+      size = human_readable(size)
+      zim_versions[perma_ref]['size'] = size
+      zim_versions[perma_ref]['tags'] = tags
+      zim_versions[perma_ref]['language'] = lang
+      zim_versions[perma_ref]['zim_date'] = date
 
+   # Write Version Map
+   if os.path.isdir(zim_version_idx_dir):
+      with open(zim_version_idx_dir + zim_version_idx_file, 'w') as fp:
+         fp.write(json.dumps(zim_versions,indent=2 ))
+         fp.close()
+   else:
+      print zim_version_idx_dir + " not found."
+      
+def get_substitution_data(perma_ref,zims_installed, path_to_id_map):
+   #reconstruct the path in the id map
+   path = 'content/' + zim_versions[perma_ref]['file_name'] + '.zim'
+   id = path_to_id_map[path]
+   item = zims_installed[id]
+   if len(item) != 0 or perma_ref == 'test':
+      mediacount = item.get('mediaCount','')
+      articlecount = item.get('articleCount','')
+      size = item.get('size','')
+      tags = item.get('tags','')
+      lang = item.get('language','')
+      if len(lang) > 2:
+         lang = lang[:2]
+      date =  item.get('date','')
+      return (articlecount,mediacount,size,tags,lang,date)
+   return ('0','0','0','0','0','0')
+
+def get_menu_def_zimnames(intended_use='zim'):
+   menu_def_dict = {}
+   os.chdir(menuDefs)
+   for filename in os.listdir('.'):
+      if fnmatch.fnmatch(filename, '*.json'):
+         try:
+            with open(filename,'r') as json_file:
+                readstr = json_file.read()
+                data = json.loads(readstr)
+         except:
+            print("failed to parse %s"%filename)
+            print(readstr)
+         if data.get('intended_use','') != 'zim':
+            continue
+         zimname = data.get('zim_name','')
+         if zimname != '':
+            menu_def_dict[data['zim_name']] = menuDefs + filename
+   return menu_def_dict
+
+def find_menuitem_from_zimname(zimname):
+   defs = get_menu_def_zimnames()
+   defs_filename = defs.get(zimname,'')
+   if defs_filename != '':
+      #print("reading menu-def:%s"%defs_filename)
+      with open(defs_filename,'r') as json_file:
+          readstr = json_file.read()
+          data = json.loads(readstr)
+          return data.get('menu_item_name','')
+   return ''
+
+def get_kiwix_catalog_item(perma_ref):
+   # Read the kiwix catalog
+   with open(KIWIX_CAT, 'r') as kiwix_cat:
+      json_data = kiwix_cat.read()
+      download = json.loads(json_data)
+      zims = download['zims']
+      for uuid in zims.keys():
+         #print("%s   %s"%(zims[uuid]['perma_ref'],perma_ref,))
+         if zims[uuid]['perma_ref'] == perma_ref:
+            return zims[uuid]
+      return {}
+
+def human_readable(num):
+    # return 3 significant digits and unit specifier
+    num = float(num)
+    units = [ '','K','M','G']
+    for i in range(4):
+        if num<10.0:
+            return "%.2f%s"%(num,units[i])
+        if num<100.0:
+            return "%.1f%s"%(num,units[i])
+        if num < 1000.0:
+            return "%.0f%s"%(num,units[i])
+        num /= 1000.0
+
+# Now start the application
 if __name__ == "__main__":
 
     # Run the main routine

roles/kolibri/tasks/main.yml

@@ -97,5 +97,5 @@
       value: "{{ kolibri_exec_path }}"
     - option: kolibri_port
       value: "{{ kolibri_http_port }}"
-    - option: enabled
+    - option: kolibri_enabled
       value: "{{ kolibri_enabled }}"

roles/network/templates/gateway/iiab-gen-iptables

@@ -58,6 +58,7 @@ calibre_port={{ calibre_port }}
 kiwix_port={{ kiwix_port }}
 kalite_server_port={{ kalite_server_port }}
 kolibri_http_port={{ kolibri_http_port }}
+cups_port={{ cups_port }}
 transmission_http_port={{ transmission_http_port }}
 transmission_peer_port={{ transmission_peer_port }}
 sugarizer_port={{ sugarizer_port }}
@@ -93,6 +94,7 @@ if [ "$services_externally_visible" == "True" ]; then
     $IPTABLES -A INPUT -p tcp --dport $kalite_server_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $kolibri_http_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $calibre_port -m state --state NEW -i $wan -j ACCEPT
+    $IPTABLES -A INPUT -p tcp --dport $cups_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $sugarizer_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $transmission_http_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $transmission_peer_port -m state --state NEW -i $wan -j ACCEPT

roles/sugarizer/tasks/main.yml

@@ -255,5 +255,5 @@
       value: Sugarizer
     - option: description
       value: '"The Sugar Learning Platform began with the famous One Laptop Per Child project, written in Python. Sugarizer is the new HTML/JavaScript implementation of Sugar, usable in most all browsers."'
-    - option: enabled
+    - option: sugarizer_enabled
       value: "{{ sugarizer_enabled }}"

roles/teamviewer/tasks/main.yml

@@ -19,7 +19,7 @@
       value: teamviewer
     - option: description
       value: '"TeamViewer - the All-In-One Software for Remote Support and Online Meetings"'
-    - option: installed
+    - option: teamviewer_install
       value: "{{ teamviewer_install }}"
-    - option: enabled
+    - option: teamviewer_enabled
       value: "{{ teamviewer_enabled }}"

vars/default_vars.yml

@@ -226,6 +226,7 @@ postgresql_enabled: False
 # Common UNIX Printing System (CUPS)
 cups_install: False
 cups_enabled: False
+cups_port: 631
 
 # Samba. Take a security audit seriously before deploying this.
 samba_install: False