1
0
Fork 0
mirror of https://github.com/iiab/iiab.git synced 2025-02-13 11:42:08 +00:00

Merge pull request #1600 from georgejhunt/vector

Regional Vector Tile selection
This commit is contained in:
A Holt 2019-04-16 11:41:49 -04:00 committed by GitHub
commit bb276dada5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 873 additions and 0 deletions

View file

@ -27,6 +27,12 @@
when: moodle_install
tags: olpc, moodle
- name: OSM_VECTOR
include_role:
name: osm-vector
when: osm_vector_install is defined and osm_vector_install
tags: osm
# UNMAINTAINED
- name: OSM
include_role:

View file

@ -0,0 +1,3 @@
menu_def_dir: '{{ doc_root }}/js-menu/menu-files/menu-defs'
osm_vector_path: '{{ content_base }}/www/osm-vector'
iiab_osm_url : http://download.iiab.io/content/OSM/vector-tiles/maplist/hidden

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,12 @@
{
"lang": "en",
"intended_use": "map",
"automatically_generated": "true",
"title": "OpenStreetMap Test Page",
"extra_html": "",
"start_url": "maplist",
"menu_item_name": "en-map_test.json",
"map_name": "en-map_test",
"logo_url": "osm.jpg",
"description": "<p>This page is installed by default during the initial installation of Internet In A Box>"
}

View file

@ -0,0 +1,216 @@
// osm_functions.js -- (non authoritative see below) src = iiab/roles/files/
// copyright 2019 George Hunt
// CAUTION -- this file is duplicate to admin-console/console/files/js/osm_fuctions.js -- please think of admin-console as authoritative
// Placed here in duplicate to ease debugging, and simplify dependences
var regionGeojson = {};
var regionList = [];
var regionInstalled = [];
var commonAssetsDir = '/common/assets/';
var mapAssetsDir = '/osm-vector/maplist/assets/';
var iiab_config_dir = '/etc/iiab/';
var onChangeFunc = "setSize";
var osmCatalog = {};
// following 2 lines an experiment to see if test page and console can be common
//var jquery = require("./assets/jquery.min");
//window.$ = window.jQuery = jquery;
function getOsmStat(){
// called during the init
console.log('in getOsmStat');
readOsmCatalog( true ); // we want checkboxes
readOsmIdx();
}
function readOsmIdx(){
//consoleLog ("in readOsmIdx");
var resp = $.ajax({
type: 'GET',
url: consoleJsonDir + 'osm-vector-idx.json',
dataType: 'json'
})
.done(function( data ) {
osmInstalled = data['regions'];
regionInstalled = [];
for (region in data['regions']) {
if (data['regions'].hasOwnProperty(region)) {
regionInstalled.push(region);
}
}
//consoleLog(osmInstalled + '');
})
.fail(jsonErrhandler);
return resp;
}
function readOsmCatalog(checkbox){
checkbox = checkbox || true;
console.log ("in readOsmCalalog");
regionList = [];
var resp = $.ajax({
type: 'GET',
url: mapAssetsDir + 'regions.json',
dataType: 'json'
})
.done(function( data ) {
regionJson = data;
osmCatalog = regionJson['regions'];
for(var key in osmCatalog){
//console.log(key + ' ' + osmCatalog[key]['title']);
osmCatalog[key]['name'] = key;
regionList.push(osmCatalog[key]);
}
})
.fail(jsonErrhandler);
return resp;
}
function renderRegionList(checkbox) { // generic
var html = "";
// order the regionList by seq number
var regions = regionList;
console.log ("in renderRegionList");
// sort on basis of seq
regions = regions.sort(function(a,b){
if (a.seq < b.seq) return -1;
else return 1;
});
//console.log(regions);
// render each region
html += '<form>';
regions.forEach((region, index) => { // now render the html
//console.log(region.title + " " +region.seq);
html += genRegionItem(region,checkbox);
});
html += '</form>';
//console.log(html);
$( "#regionlist" ).html(html);
}
function genRegionItem(region,checkbox) {
var html = "";
console.log("in genRegionItem: " + region.name);
var itemId = region.title;
var ksize = region.size / 1000;
//console.log(html);
html += '<div class="extract" data-region={"name":"' + region.name + '"}>';
html += ' <label>';
if ( checkbox ) {
if (selectedOsmItems.indexOf(region.name) != -1)
checked = 'checked';
else
checked = '';
html += '<input type="checkbox" name="' + region.name + '"';
html += ' onChange="updateOsmSpace(this)" ' + checked + '>';
}
html += itemId;
if ( checkbox ) { html += '</input>';};
html += '</label>'; // end input
html += ' Size: ' + readableSize(ksize);
html += '</div>';
//console.log(html);
return html;
}
function instOsmItem(name) {
var command = "INST-OSM-VECT-SET";
var cmd_args = {};
cmd_args['osm_vect_id'] = name;
cmd = command + " " + JSON.stringify(cmd_args);
sendCmdSrvCmd(cmd, genericCmdHandler);
osmDownloading.push(name);
if ( osmWip.indexOf(name) != -1 )
osmWip.push(osmCatalog[name]);
console.log('osmWip: ' + osmWip);
return true;
}
function jsonErrhandler (jqXHR, textStatus, errorThrown)
{
// only handle json parse errors here, others in ajaxErrHandler
if (textStatus == "parserror") {
//alert ("Json Errhandler: " + textStatus + ", " + errorThrown);
displayServerCommandStatus("Json Errhandler: " + textStatus + ", " + errorThrown);
}
//consoleLog("In Error Handler logging jqXHR");
console.log(textStatus);
console.log(errorThrown);
console.log(jqXHR);
return false;
}
function readableSize(kbytes) {
if (kbytes == 0)
return "0";
var bytes = 1024 * kbytes;
var s = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'];
var e = Math.floor(Math.log(bytes) / Math.log(1024));
return (bytes / Math.pow(1024, e)).toFixed(2) + " " + s[e];
}
function updateOsmSpace(cb){
console.log("in updateOsmSpace" + cb);
var region = cb.name;
updateOsmSpaceUtil(region, cb.checked);
}
function updateOsmSpaceUtil(region, checked){
var size = parseInt(osmCatalog[region].size);
var modIdx = selectedOsmItems.indexOf(region);
if (checked){
if (regionInstalled.indexOf(region) == -1){ // only update if not already installed mods
sysStorage.osm_selected_size += size;
selectedOsmItems.push(region);
}
}
else {
if (modIdx != -1){
sysStorage.osm_selected_size -= size;
selectedOsmItems.splice(modIdx, 1);
}
}
displaySpaceAvail();
}
/*
function totalSpace(){
// obsolete but perhaps useful in debugging since it worked
var sum = 0;
$( ".extract" ).each(function(ind,elem){
var data = JSON.parse($(this).attr('data-region'));
var region = data.name;
var size = parseInt(osmCatalog[region]['size']);
var chk = $( this ).find(':checkbox').prop("checked") == true;
if (chk && typeof size !== 'undefined')
sum += size;
});
var ksize = sum / 1000;
$( "#osmDiskSpace" ).html(readableSize(ksize));
}
$( '#instOsmRegion').on('click', function(evnt){
readOsmCatalog();
osm.render();
});
*/
function renderOsm(){
console.log('in renderOsm');
window.map.setTarget($("#osm-container")[0]);
window.map.render();
renderRegionList(true);
}
function initOsm(){
var dummy = 0;
sysStorage.osm_selected_size = 0;
$.when(readOsmCatalog(true)).then(renderRegionList);
}

View file

@ -0,0 +1,4 @@
<head>
<meta http-equiv="refresh" content="0; URL=/osm-vector/maplist/" />
</head>

View file

@ -0,0 +1,101 @@
- name: Make sure the osm-vector directory exists
file:
path: '{{ osm_vector_path }}/maplist/assets'
state: directory
owner: '{{ apache_user }}'
group: '{{ apache_user }}'
mode: '0755'
- name: Fetch the catalog for osm maps
get_url:
url: "{{ iiab_osm_url }}/assets/regions.json"
dest: '{{ osm_vector_path }}/maplist/assets/'
- name: Create a link to osm catalog in /common/assets
file:
src: "{{ osm_vector_path }}/maplist/assets/regions.json"
dest: "{{ doc_root }}/common/assets/regions.json"
state: link
- name: Fetch the javascript bundle with openlayers for test page
get_url:
url: "{{ iiab_osm_url }}/../main.js"
dest: '{{ osm_vector_path }}/maplist/'
- name: Fetch the index.html for test page
template:
src: "index.html"
dest: '{{ osm_vector_path }}/maplist/index.html'
# Bboxes are currently square. But geofabrik has non-rectangular bboxes
# So bring the bounding box definition from cloud (bboxes.geojson is big)
- name: Fetch the bounding box description for osm maps
get_url:
url: "{{ iiab_osm_url }}/assets/bboxes.geojson"
dest: '{{ osm_vector_path }}/maplist/assets/'
- name: Install a package that helps with geojson
package:
name:
- python-geojson
state: present
- name: Install the script to update osm catalog
template:
src: iiab-update-osm
dest: /usr/bin/iiab-update-osm
mode: "0755"
- name: Run the script that does osm-vector housekeeping
shell: /usr/bin/iiab-update-osm
- name: Copy the Countries geojson to assets
copy:
src: countries.json
dest: '{{ osm_vector_path }}/maplist/assets'
# It is too complicated to use a single file for both iiab and admin-console
- name: Copy the duplicated javascript to assets
copy:
src: osm_functions.js
dest: '{{ osm_vector_path }}/maplist/assets'
- name: Install /etc/{{ apache_config_dir }}/osm-vect.conf from template
template:
src: osm-vector.conf
dest: "/etc/{{ apache_config_dir }}/osm-vector.conf"
- name: Create symlink osm-vector.conf from sites-enabled to sites-available (debuntu, not nec for redhat)
file:
src: /etc/apache2/sites-available/osm-vector.conf
path: /etc/apache2/sites-enabled/osm-vector.conf
state: link
when: osm_vector_enabled and is_debuntu
- name: Remove symlink /etc/apache2/sites-enabled/osm-vector.conf (debuntu)
file:
path: /etc/apache2/sites-enabled/osm-vector.conf
state: absent
when: not osm_vector_enabled and is_debuntu
- name: Copy the redirect to the test page -- delete this if more than one map
copy:
src: test-index.redirect
dest: "{{ osm_vector_path }}/index.html"
- name: Create a directory for map menu-def
file:
path: '{{ menu_def_dir }}'
state: directory
- name: Determine status of splash menu def
stat:
path: '{{ menu_def_dir }}/en-map_test.json'
ignore_errors: True
register: menu_def
- name: Do not overwrite if it already exists
copy:
src: en-map_test.json
dest: '{{ menu_def_dir }}/en-map_test.json'
when: menu_def.stat.exists is defined and not menu_def.stat.exists

View file

@ -0,0 +1,189 @@
#!/usr/bin/env python
# Scan the osm-vector directory, update the osm-vector-idx.json, add menu-defs
from geojson import Feature, Point, FeatureCollection, Polygon
import geojson
import json
import os
import sys
import fnmatch
import re
IIAB_PATH='/etc/iiab'
if not IIAB_PATH in sys.path:
sys.path.append(IIAB_PATH)
from iiab_env import get_iiab_env
SCRIPT_DIR = '/opt/admin/cmdsrv/scripts'
if not SCRIPT_DIR in sys.path:
sys.path.append(SCRIPT_DIR)
if os.path.exists(os.path.join(SCRIPT_DIR,'iiab_update_menus.py')):
import iiab_update_menus as menus
console_installed = True
else:
console_installed = False
doc_root = get_iiab_env('WWWROOT')
menuDefs = doc_root + "/js-menu/menu-files/menu-defs/"
osm_vector_idx_dir = doc_root + "/common/assets"
#map_doc_root = '{{ osm_vector_path }}'
map_doc_root = '/library/www/osm-vector'
# map_catalog will be global, assumed always available
map_catalog = {}
map_menu_def_list = []
def main():
global map_menu_def_list
get_map_catalog()
#print(json.dumps(map_catalog,indent=2))
map_menu_def_list = get_menu_def_names()
print(json.dumps(map_menu_def_list,indent=2))
installed_maps = get_installed_regions()
print(installed_maps)
write_osm_vector_idx(installed_maps)
# For installed regions, check that a menu def exists, and it's on home page
for fname in installed_maps:
region = extract_region_from_filename(fname)
print('checking for %s region'%region)
if region == 'maplist': # it is the splash page, display only if no others
menu_ref = 'en-map_test'
item = { "perma_ref" : "en-map_test" }
if len(installed_maps) == 1:
menus.update_menu_json(menu_ref)
return
else:
item = map_catalog['regions'][region]
menu_ref = item['perma_ref']
if not (region in map_menu_def_list):
print('creating menu def for %s'%item['perma_ref'])
create_menu_def(region,item['perma_ref'] + '.json')
if fetch_menu_json_value('autoupdate_menu'):
print('autoudate of menu items is enabled:%s. Adding %s'%(\
fetch_menu_json_value('autoupdate_menu'),region,))
# verify this menu def is on home page
menus.update_menu_json(menu_ref)
def get_map_catalog():
global map_catalog
input_json = map_doc_root + '/maplist/assets/regions.json'
with open(input_json,'r') as regions:
reg_str = regions.read()
map_catalog = json.loads(reg_str)
#print(json.dumps(map_catalog,indent=2))
def get_menu_def_names(intended_use='map'):
menu_def_list =[]
os.chdir(menuDefs)
for filename in os.listdir('.'):
if fnmatch.fnmatch(filename, '*.json'):
try:
with open(filename,'r') as json_file:
readstr = json_file.read()
data = json.loads(readstr)
except:
print("failed to parse %s"%filename)
print(readstr)
if data.get('intended_use','') != intended_use:
continue
map_name = data.get('name','')
if map_name != '':
menu_def_list.append(data['name'])
return menu_def_list
def get_installed_regions():
installed = []
os.chdir(map_doc_root)
for filename in os.listdir('.'):
if fnmatch.fnmatch(filename, '??-osm-omt*'):
region = re.sub(r'^..-osm-omt_(.*)',r'\1',filename)
installed.append(region)
# add the splash page if no other maps are present
if len(installed) == 0:
installed.append('maplist')
return installed
def write_osm_vector_idx(installed_maps):
map_dict ={}
idx_dict = {}
for fname in installed_maps:
region = extract_region_from_filename(fname)
if map == 'maplist': continue # not a real region
map_dict = map_catalog['regions'].get(region,'')
if map_dict == '': continue
# Create the idx file in format required bo js-menu system
item = map_dict['perma_ref']
idx_dict[item] = {}
idx_dict[item]['file_name'] = os.path.basename(map_dict['url'][:-4])
idx_dict[item]['menu_item'] = map_dict['perma_ref']
idx_dict[item]['size'] = map_dict['size']
idx_dict[item]['date'] = map_dict['date']
idx_dict[item]['region'] = region
idx_dict[item]['language'] = map_dict['perma_ref'][:2]
with open(osm_vector_idx_dir + '/osm_version_idx.json','w') as idx:
idx.write(json.dumps(idx_dict,indent=2))
def create_menu_def(region,default_name,intended_use='map'):
item = map_catalog['regions'][region]
if len(item.get('language','')) > 2:
lang = item['language'][:2]
else: # default to english
lang = 'en'
filename = lang + '-' + item['perma_ref'] + '.json'
# create a stub for this zim
menuDef = {}
default_logo = 'osm.jpg'
menuDef["intended_use"] = "map"
menuDef["lang"] = lang
menuDef["logo_url"] = default_logo
menuitem = lang + '-' + item['perma_ref']
menuDef["menu_item_name"] = default_name
menuDef["title"] = "OpenStreetMap: 18 Levels of Zoom for <b> " + item.get('title','ERROR') + '</b>'
menuDef["map_name"] = item['perma_ref']
menuDef["file_name"] = lang + '-osm-omt_' + region + '_' + \
os.path.basename(item['url'])[:-4]
menuDef["description"] = '<p>Resolution of the Whole World to 5 KM. OpenStreetMap data for <b>' + item.get('title','') + '</b> with details down to 5 Meters</p>'
menuDef["extra_html"] = ""
menuDef["automatically_generated"] = "true"
if not os.path.isfile(menuDefs + default_name): # logic to here can still overwrite existing menu def
print("creating %s"%menuDefs + default_name)
with open(menuDefs + default_name,'w') as menufile:
menufile.write(json.dumps(menuDef,indent=2))
return default_name[:-5]
def human_readable(num):
# return 3 significant digits and unit specifier
num = float(num)
units = [ '','K','M','G']
for i in range(4):
if num<10.0:
return "%.2f%s"%(num,units[i])
if num<100.0:
return "%.1f%s"%(num,units[i])
if num < 1000.0:
return "%.0f%s"%(num,units[i])
num /= 1000.0
def fetch_menu_json_value(key):
with open( doc_root + '/home/menu.json','r') as menudef:
data = json.loads(menudef.read())
return data.get(key,'')
def extract_region_from_filename(fname):
substitutions = { "north": "north_america",\
"central": "central_america",\
"southeast": "southeast_asia",
"south": "south_america" }
# wish I had used - as separator between key and date
nibble = fname.split('_')[0]
nibble = substitutions.get(nibble,nibble)
return(nibble)
if __name__ == '__main__':
if console_installed:
main()

View file

@ -0,0 +1,57 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Osm-Vector Splash Page</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Bootstrap -->
<link href="/common/css/bootstrap.min.css" rel="stylesheet">
<script src='/common/js/jquery.min.js'></script>
</head>
<body>
<h2>Vector Maps Test Page -- Use The Admin Console to Download a Selected Region</h2>
<p>The IIAB vector maps are based upon OpenStreetMap Data. Each map has general geographic information that covers the <b>whole world </b>with a resolution of about 5 KM. The detailed regions provide data about items that are 5 meters apart. </p>
<div class="row">
<div class="col-sm-12"><a href="/admin"><button id="INST-MODS" type="button" class="btn btn-lg btn-primary">Go To Admin Console</button></a></div>
</div>
<div id="osmRegionSelection" >
<style>
#map-container{
background-color: #b2d3f8;
float: right;
width: 70%;
right: 10px;
height: 500px;
}
body {
padding: 40px;
}
</style>
<div id="map-container"></div>
</div>
<div id='regionlist'>placeholdr</div>
</div> <!-- End instOsmRegion Submenu Option Pane -->
<script>
function readableSize(kbytes) {
if (kbytes == 0)
return "0";
var bytes = 1024 * kbytes;
var s = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'];
var e = Math.floor(Math.log(bytes) / Math.log(1024));
return (bytes / Math.pow(1024, e)).toFixed(2) + " " + s[e];
}
</script>
<script src="/osm-vector/maplist/assets/osm_functions.js"></script>
<script>
window.$ = window.$ = jQuery;
$.when(readOsmCatalog()).done(function() {
renderRegionList(false);
});
</script>
<script type="text/javascript" src="/osm-vector/maplist/main.js"></script></body>
</html>

View file

@ -0,0 +1,89 @@
import {Fill, Stroke, Style} from 'ol/style';
import 'ol/ol.css';
import GeoJSON from 'ol/format/GeoJSON';
import Map from 'ol/Map';
import VectorLayer from 'ol/layer/Vector';
import VectorSource from 'ol/source/Vector';
import View from 'ol/View';
//import XYZSource from 'ol/source/XYZ';
//import MVT from 'ol/format/MVT';
// a global variable to control which features are shown
var show = {};
var mapData = "/common/assets";
var map = new Map({
target: 'map-container',
layers: [
new VectorLayer({
source: new VectorSource({
format: new GeoJSON(),
url: mapData + '/countries.json'
}),
style: new Style({
fill: new Fill({
color: 'rgb(219, 180, 131)'
}),
stroke: new Stroke({
color: 'white'
})
})
}),
],
view: new View({
center: [0, 0],
zoom: 2
})
});
var setBoxStyle = function(feature) {
var name = feature.get("name");
//alert(keys+'');
if (typeof show !== 'undefined' &&
show != null && name == show) {
return new Style({
fill: new Fill({
color: 'rgba(67, 163, 46, 0.2)'
}),
stroke: new Stroke({
color: 'rgba(67, 163, 46, 1)',
width: 2
})
})
} else {
return new Style({
fill: new Fill({
color: 'rgba(255,255,255,.10)'
}),
stroke: new Stroke({
color: 'rgba(255,255,255,.3)'
})
})
}
}
var boxLayer = new VectorLayer({
source: new VectorSource({
format: new GeoJSON(),
url: mapData + '/bboxes.geojson'
}),
id: 'boxes',
style: setBoxStyle
});
map.addLayer(boxLayer);
$( document ).on("mouseover",".extract",function(){
var data = JSON.parse($(this).attr('data-region'));
show = data.name;
//setBoxStyle();
boxLayer.changed();
});
$( document ).on("mouseout",".extract",function(){
var data = JSON.parse($(this).attr('data-region'));
show = '';
boxLayer.changed();
});

View file

@ -0,0 +1,8 @@
# For downloadable regional vector tilesets
Alias /maps {{ osm_vector_path }}
Alias /osm-vector {{ osm_vector_path }}
<Directory {{ osm_vector_path }}>
Options Indexes FollowSymLinks
AllowOverride All
Require all granted
</Directory>

View file

@ -15,6 +15,7 @@ pip_packages_dir: "{{ iiab_base }}/pip-packages"
yum_packages_dir: "{{ iiab_base }}/yum-packages"
downloads_dir: "{{ iiab_base }}/downloads"
iiab_download_url: http://download.iiab.io/packages
iiab_osm_url : http://download.iiab.io/content/OSM/vector-tiles/maplist/hidden
content_base: "/library"
doc_base: "{{ content_base }}/www"
@ -350,6 +351,11 @@ mongodb_install: False
mongodb_enabled: False
mongodb_port: 27018
# Regional OSM vector maps take much less disk space than image versions
osm_vector_install: True
osm_vector_enabled: True
osm_vector_path: '{{ content_base }}/www/osm-vector'
# roles/sugarizer/meta/main.yml auto-invokes 2 above prereqs: mongodb & nodejs
# Might stall MongoDB on Power Failure: github.com/xsce/xsce/issues/879
# Sugarizer 1.0.1+ strategies to solve? github.com/iiab/iiab/pull/957