166 lines
8.7 KiB
Python
Executable file
166 lines
8.7 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2018 Mike "KemoNine" Crosson
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import subprocess, os, zipfile, shutil, fileinput, glob
|
|
from . import poly
|
|
|
|
FNULL = open(os.devnull, 'w')
|
|
|
|
SOURCE_MAP_POSTFIX = '-latest.osm.pbf'
|
|
|
|
def setup_working_dirs(base_path, current_timestamp, output_map_name):
|
|
print('Creating working directories as needed')
|
|
if not os.path.exists(os.path.join(base_path, 'tmp')):
|
|
os.makedirs(os.path.join(base_path, 'tmp'))
|
|
if not os.path.exists(os.path.join(base_path, 'out')):
|
|
os.makedirs(os.path.join(base_path, 'out'))
|
|
output_dir = current_timestamp + '-' + output_map_name
|
|
if not os.path.exists(os.path.join(base_path, 'out', output_dir)):
|
|
os.makedirs(os.path.join(base_path, 'out', output_dir))
|
|
|
|
def prep_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name):
|
|
print('Prepping land/sea polygons (one prep per map in list(s))')
|
|
output_dir = current_timestamp + '-' + output_map_name
|
|
polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
|
|
polys_zip = zipfile.ZipFile(os.path.join(maps_dir, 'land-polygons-split-4326.zip'))
|
|
|
|
for alist in map_lists:
|
|
with open(alist, 'r') as maps:
|
|
for line in maps:
|
|
map_polys_dir = os.path.join(polys_output_dir, line.strip().split('/')[-1])
|
|
print(' Extracting to: ' + map_polys_dir)
|
|
os.makedirs(map_polys_dir)
|
|
polys_zip.extractall(map_polys_dir)
|
|
|
|
polys_zip.close()
|
|
|
|
def process_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram):
|
|
print('Processing land/sea polygons (one process per map in list(s))')
|
|
output_dir = current_timestamp + '-' + output_map_name
|
|
polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
|
|
cached_maps_dir = os.path.abspath(maps_dir)
|
|
for alist in map_lists:
|
|
with open(alist, 'r') as maps:
|
|
for line in maps:
|
|
current_map = line.strip().split('/')[-1]
|
|
print(' Processing: ' + current_map)
|
|
bounds = poly.calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly'))
|
|
subprocess.run(['ogr2ogr', '-overwrite', '-progress', '-skipfailures', '-clipsrc', str(bounds['left']), str(bounds['bottom']), str(bounds['right']), str(bounds['top']), os.path.join(polys_output_dir, current_map + '.shp'), os.path.join(polys_output_dir, current_map, 'land-polygons-split-4326', 'land_polygons.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
|
|
subprocess.run(['python2', os.path.join(base_path, 'shape2osm.py'), '-l', os.path.join(polys_output_dir, current_map), os.path.join(polys_output_dir, current_map + '.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
|
|
shutil.copyfile(os.path.join(base_path, 'sea.osm'), os.path.join(polys_output_dir, current_map + '_sea.osm'))
|
|
with fileinput.FileInput(os.path.join(polys_output_dir, current_map + '_sea.osm'), inplace=True, backup='.bak') as sea:
|
|
for line in sea:
|
|
updated_line = line.replace('$BOTTOM', str(bounds['bottom']))
|
|
updated_line = updated_line.replace('$LEFT', str(bounds['left']))
|
|
updated_line = updated_line.replace('$TOP', str(bounds['top']))
|
|
updated_line = updated_line.replace('$RIGHT', str(bounds['right']))
|
|
print(updated_line, end='')
|
|
|
|
map_list = []
|
|
for alist in map_lists:
|
|
with open(alist, 'r') as maps:
|
|
for line in maps:
|
|
map_list.append(line.strip().split('/')[-1])
|
|
|
|
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
|
|
env = os.environ.copy()
|
|
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
|
|
|
|
print('Processing land/sea polys using osmosis')
|
|
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
|
|
for amap in map_list:
|
|
osmosis_cmd.extend(['--rb', 'file=' + os.path.join(maps_dir, amap + SOURCE_MAP_POSTFIX)])
|
|
for apolymap in glob.glob(os.path.join(polys_output_dir, amap + '*.osm')):
|
|
print(' Found poly map : ' + apolymap)
|
|
osmosis_cmd.extend(['--rx', 'file=' + apolymap, '--s', '--m'])
|
|
osmosis_cmd.extend(['--log-progress'])
|
|
for x in range(0, len(map_list) - 1):
|
|
osmosis_cmd.append('--merge')
|
|
osmosis_cmd.extend(['--log-progress'])
|
|
osmosis_cmd.extend(['--wb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
|
|
osmosis_cmd.extend(['omitmetadata=true'])
|
|
cmd = subprocess.Popen(osmosis_cmd, env=env)
|
|
cmd.wait()
|
|
|
|
def process_tag_transform(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram, tag_transform):
|
|
print('Transforming tags prior to final map writing')
|
|
output_dir = current_timestamp + '-' + output_map_name
|
|
cached_maps_dir = os.path.abspath(maps_dir)
|
|
|
|
poly_map = os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'
|
|
|
|
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
|
|
env = os.environ.copy()
|
|
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
|
|
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
|
|
osmosis_cmd.extend(['--rb', 'file=' + poly_map])
|
|
osmosis_cmd.extend(['--tag-transform', 'file=' + os.path.abspath(tag_transform)])
|
|
osmosis_cmd.extend(['--log-progress'])
|
|
osmosis_cmd.extend(['--wb', 'file=' + poly_map + 'tt.pbf'])
|
|
osmosis_cmd.extend(['omitmetadata=true'])
|
|
cmd = subprocess.Popen(osmosis_cmd, env=env)
|
|
cmd.wait()
|
|
|
|
shutil.move(poly_map, poly_map + '.orig')
|
|
shutil.move(poly_map + 'tt.pbf', poly_map)
|
|
|
|
def osmosis_main(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram, tag_mapping):
|
|
output_dir = current_timestamp + '-' + output_map_name
|
|
|
|
bounds = poly.calculate_total_bounds(maps_dir, map_lists)
|
|
|
|
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
|
|
env = os.environ.copy()
|
|
env['JAVACMD_OPTIONS'] = '-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
|
|
|
|
print('-----------------------------')
|
|
print('Processing maps using osmosis')
|
|
print('-----------------------------')
|
|
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
|
|
osmosis_cmd.extend(['--rb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
|
|
osmosis_cmd.extend(['--log-progress'])
|
|
osmosis_cmd.extend(['--mapfile-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.map'])
|
|
if use_ram:
|
|
osmosis_cmd.extend(['type=ram'])
|
|
else:
|
|
osmosis_cmd.extend(['type=hd'])
|
|
osmosis_cmd.extend(['bbox=' + str(bounds['bottom']) + ',' + str(bounds['left']) + ',' + str(bounds['top']) + ',' + str(bounds['right'])])
|
|
osmosis_cmd.extend(['threads=1', 'map-start-zoom=8'])
|
|
if tag_mapping is not None:
|
|
osmosis_cmd.extend(['tag-conf-file=' + os.path.abspath(tag_mapping)])
|
|
cmd = subprocess.Popen(osmosis_cmd, env=env)
|
|
cmd.wait()
|
|
|
|
def osmosis_pois(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space):
|
|
output_dir = current_timestamp + '-' + output_map_name
|
|
|
|
map_list = []
|
|
for alist in map_lists:
|
|
with open(alist, 'r') as maps:
|
|
for line in maps:
|
|
map_list.append(os.path.join(maps_dir, line.strip().split('/')[-1]+SOURCE_MAP_POSTFIX))
|
|
|
|
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
|
|
env = os.environ.copy()
|
|
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
|
|
print('Processing POIs using osmosis')
|
|
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
|
|
for amap in map_list:
|
|
osmosis_cmd.extend(['--rb', 'file=' + amap])
|
|
for x in range(0, len(map_list) - 1):
|
|
osmosis_cmd.append('--merge')
|
|
osmosis_cmd.extend(['--poi-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')])
|
|
cmd = subprocess.Popen(osmosis_cmd, env=env)
|
|
cmd.wait()
|