#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2018 Mike "KemoNine" Crosson # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import subprocess, os, zipfile, shutil, fileinput, glob from . import poly FNULL = open(os.devnull, 'w') SOURCE_MAP_POSTFIX = '-latest.osm.pbf' def setup_working_dirs(base_path, current_timestamp, output_map_name): print('Creating working directories as needed') if not os.path.exists(os.path.join(base_path, 'tmp')): os.makedirs(os.path.join(base_path, 'tmp')) if not os.path.exists(os.path.join(base_path, 'out')): os.makedirs(os.path.join(base_path, 'out')) output_dir = current_timestamp + '-' + output_map_name if not os.path.exists(os.path.join(base_path, 'out', output_dir)): os.makedirs(os.path.join(base_path, 'out', output_dir)) def prep_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name): print('Prepping land/sea polygons (one prep per map in list(s))') output_dir = current_timestamp + '-' + output_map_name polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys') polys_zip = zipfile.ZipFile(os.path.join(maps_dir, 'land-polygons-split-4326.zip')) for alist in map_lists: with open(alist, 'r') as maps: for line in maps: map_polys_dir = os.path.join(polys_output_dir, line.strip().split('/')[-1]) print(' Extracting to: ' + map_polys_dir) os.makedirs(map_polys_dir) polys_zip.extractall(map_polys_dir) polys_zip.close() def process_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram): print('Processing land/sea polygons (one process per map in list(s))') output_dir = current_timestamp + '-' + output_map_name polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys') cached_maps_dir = os.path.abspath(maps_dir) for alist in map_lists: with open(alist, 'r') as maps: for line in maps: current_map = line.strip().split('/')[-1] print(' Processing: ' + current_map) bounds = poly.calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly')) subprocess.run(['ogr2ogr', '-overwrite', '-progress', '-skipfailures', '-clipsrc', str(bounds['left']), str(bounds['bottom']), str(bounds['right']), str(bounds['top']), os.path.join(polys_output_dir, current_map + '.shp'), os.path.join(polys_output_dir, current_map, 'land-polygons-split-4326', 'land_polygons.shp')], stdout=FNULL, stderr=subprocess.STDOUT) subprocess.run(['python2', os.path.join(base_path, 'shape2osm.py'), '-l', os.path.join(polys_output_dir, current_map), os.path.join(polys_output_dir, current_map + '.shp')], stdout=FNULL, stderr=subprocess.STDOUT) shutil.copyfile(os.path.join(base_path, 'sea.osm'), os.path.join(polys_output_dir, current_map + '_sea.osm')) with fileinput.FileInput(os.path.join(polys_output_dir, current_map + '_sea.osm'), inplace=True, backup='.bak') as sea: for line in sea: updated_line = line.replace('$BOTTOM', str(bounds['bottom'])) updated_line = updated_line.replace('$LEFT', str(bounds['left'])) updated_line = updated_line.replace('$TOP', str(bounds['top'])) updated_line = updated_line.replace('$RIGHT', str(bounds['right'])) print(updated_line, end='') map_list = [] for alist in map_lists: with open(alist, 'r') as maps: for line in maps: map_list.append(line.strip().split('/')[-1]) # Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) env = os.environ.copy() env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) print('Processing land/sea polys using osmosis') osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] for amap in map_list: osmosis_cmd.extend(['--rb', 'file=' + os.path.join(maps_dir, amap + SOURCE_MAP_POSTFIX)]) for apolymap in glob.glob(os.path.join(polys_output_dir, amap + '*.osm')): print(' Found poly map : ' + apolymap) osmosis_cmd.extend(['--rx', 'file=' + apolymap, '--s', '--m']) osmosis_cmd.extend(['--log-progress']) for x in range(0, len(map_list) - 1): osmosis_cmd.append('--merge') osmosis_cmd.extend(['--log-progress']) osmosis_cmd.extend(['--wb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf']) osmosis_cmd.extend(['omitmetadata=true']) cmd = subprocess.Popen(osmosis_cmd, env=env) cmd.wait() def process_tag_transform(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram, tag_transform): print('Transforming tags prior to final map writing') output_dir = current_timestamp + '-' + output_map_name cached_maps_dir = os.path.abspath(maps_dir) poly_map = os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf' # Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) env = os.environ.copy() env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] osmosis_cmd.extend(['--rb', 'file=' + poly_map]) osmosis_cmd.extend(['--tag-transform', 'file=' + os.path.abspath(tag_transform)]) osmosis_cmd.extend(['--log-progress']) osmosis_cmd.extend(['--wb', 'file=' + poly_map + 'tt.pbf']) osmosis_cmd.extend(['omitmetadata=true']) cmd = subprocess.Popen(osmosis_cmd, env=env) cmd.wait() shutil.move(poly_map, poly_map + '.orig') shutil.move(poly_map + 'tt.pbf', poly_map) def osmosis_main(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram, tag_mapping): output_dir = current_timestamp + '-' + output_map_name bounds = poly.calculate_total_bounds(maps_dir, map_lists) # Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) env = os.environ.copy() env['JAVACMD_OPTIONS'] = '-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) print('-----------------------------') print('Processing maps using osmosis') print('-----------------------------') osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] osmosis_cmd.extend(['--rb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf']) osmosis_cmd.extend(['--log-progress']) osmosis_cmd.extend(['--mapfile-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.map']) if use_ram: osmosis_cmd.extend(['type=ram']) else: osmosis_cmd.extend(['type=hd']) osmosis_cmd.extend(['bbox=' + str(bounds['bottom']) + ',' + str(bounds['left']) + ',' + str(bounds['top']) + ',' + str(bounds['right'])]) osmosis_cmd.extend(['threads=1', 'map-start-zoom=8']) if tag_mapping is not None: osmosis_cmd.extend(['tag-conf-file=' + os.path.abspath(tag_mapping)]) cmd = subprocess.Popen(osmosis_cmd, env=env) cmd.wait() def osmosis_pois(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space): output_dir = current_timestamp + '-' + output_map_name map_list = [] for alist in map_lists: with open(alist, 'r') as maps: for line in maps: map_list.append(os.path.join(maps_dir, line.strip().split('/')[-1]+SOURCE_MAP_POSTFIX)) # Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) env = os.environ.copy() env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) print('Processing POIs using osmosis') osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] for amap in map_list: osmosis_cmd.extend(['--rb', 'file=' + amap]) for x in range(0, len(map_list) - 1): osmosis_cmd.append('--merge') osmosis_cmd.extend(['--poi-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')]) cmd = subprocess.Popen(osmosis_cmd, env=env) cmd.wait()