osm_map_processing/pymapsforge/process.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess, os, zipfile, shutil, fileinput, glob
from . import poly

FNULL = open(os.devnull, 'w')

SOURCE_MAP_POSTFIX = '-latest.osm.pbf'

def setup_working_dirs(base_path, current_timestamp, output_map_name):
	print('Creating working directories as needed')
	if not os.path.exists(os.path.join(base_path, 'tmp')):
		os.makedirs(os.path.join(base_path, 'tmp'))
	if not os.path.exists(os.path.join(base_path, 'out')):
		os.makedirs(os.path.join(base_path, 'out'))
	output_dir = current_timestamp + '-' + output_map_name
	if not os.path.exists(os.path.join(base_path, 'out', output_dir)):
		os.makedirs(os.path.join(base_path, 'out', output_dir))

def prep_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name):
	print('Prepping land/sea polygons (one prep per map in list(s))')
	output_dir = current_timestamp + '-' + output_map_name
	polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
	polys_zip = zipfile.ZipFile(os.path.join(maps_dir, 'land-polygons-split-4326.zip'))
	
	for alist in map_lists:
		with open(alist, 'r') as maps:
			for line in maps:
				map_polys_dir = os.path.join(polys_output_dir, line.strip().split('/')[-1])
				print('    Extracting to: ' + map_polys_dir)
				os.makedirs(map_polys_dir)
				polys_zip.extractall(map_polys_dir)
 
	polys_zip.close()

def process_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram):
	print('Processing land/sea polygons (one process per map in list(s))')
	output_dir = current_timestamp + '-' + output_map_name
	polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
	cached_maps_dir = os.path.abspath(maps_dir)
	for alist in map_lists:
		with open(alist, 'r') as maps:
			for line in maps:
				current_map = line.strip().split('/')[-1]
				print('    Processing: ' + current_map)
				bounds = poly.calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly'))
				subprocess.run(['ogr2ogr', '-overwrite', '-progress', '-skipfailures', '-clipsrc', str(bounds['left']), str(bounds['bottom']), str(bounds['right']), str(bounds['top']), os.path.join(polys_output_dir, current_map + '.shp'), os.path.join(polys_output_dir, current_map, 'land-polygons-split-4326', 'land_polygons.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
				subprocess.run(['python2', os.path.join(base_path, 'shape2osm.py'), '-l', os.path.join(polys_output_dir, current_map), os.path.join(polys_output_dir, current_map + '.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
				shutil.copyfile(os.path.join(base_path, 'sea.osm'),  os.path.join(polys_output_dir, current_map + '_sea.osm'))
				with fileinput.FileInput(os.path.join(polys_output_dir, current_map + '_sea.osm'), inplace=True, backup='.bak') as sea:
					for line in sea:
						updated_line = line.replace('$BOTTOM', str(bounds['bottom']))
						updated_line = updated_line.replace('$LEFT', str(bounds['left']))
						updated_line = updated_line.replace('$TOP', str(bounds['top']))
						updated_line = updated_line.replace('$RIGHT', str(bounds['right']))
						print(updated_line, end='')

	map_list = []
	for alist in map_lists:
		with open(alist, 'r') as maps:
			for line in maps:
				map_list.append(line.strip().split('/')[-1])

	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
	env = os.environ.copy()
	env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)

	print('Processing land/sea polys using osmosis')
	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
	for amap in map_list:
		osmosis_cmd.extend(['--rb', 'file=' + os.path.join(maps_dir, amap + SOURCE_MAP_POSTFIX)])
		for apolymap in glob.glob(os.path.join(polys_output_dir, amap + '*.osm')):
			print('    Found poly map : ' + apolymap)
			osmosis_cmd.extend(['--rx', 'file=' + apolymap, '--s', '--m'])
	osmosis_cmd.extend(['--log-progress'])
	for x in range(0, len(map_list) - 1):
		osmosis_cmd.append('--merge')
	osmosis_cmd.extend(['--log-progress'])
	osmosis_cmd.extend(['--wb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
	osmosis_cmd.extend(['omitmetadata=true'])
	cmd = subprocess.Popen(osmosis_cmd, env=env)
	cmd.wait()

def process_tag_transform(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram, tag_transform):
	print('Transforming tags prior to final map writing')
	output_dir = current_timestamp + '-' + output_map_name
	cached_maps_dir = os.path.abspath(maps_dir)

	poly_map = os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'

	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
	env = os.environ.copy()
	env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
	osmosis_cmd.extend(['--rb', 'file=' + poly_map])
	osmosis_cmd.extend(['--tag-transform', 'file=' + os.path.abspath(tag_transform)])
	osmosis_cmd.extend(['--log-progress'])
	osmosis_cmd.extend(['--wb', 'file=' + poly_map + 'tt.pbf'])
	osmosis_cmd.extend(['omitmetadata=true'])
	cmd = subprocess.Popen(osmosis_cmd, env=env)
	cmd.wait()

	shutil.move(poly_map, poly_map + '.orig')
	shutil.move(poly_map + 'tt.pbf', poly_map)

def osmosis_main(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram, tag_mapping):
	output_dir = current_timestamp + '-' + output_map_name

	bounds = poly.calculate_total_bounds(maps_dir, map_lists)

	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
	env = os.environ.copy()
	env['JAVACMD_OPTIONS'] = '-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
	
	print('-----------------------------')
	print('Processing maps using osmosis')
	print('-----------------------------')
	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
	osmosis_cmd.extend(['--rb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
	osmosis_cmd.extend(['--log-progress'])
	osmosis_cmd.extend(['--mapfile-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.map'])
	if use_ram:
		osmosis_cmd.extend(['type=ram'])
	else:
		osmosis_cmd.extend(['type=hd'])
	osmosis_cmd.extend(['bbox=' + str(bounds['bottom']) + ',' + str(bounds['left']) + ',' + str(bounds['top']) + ',' + str(bounds['right'])])
	osmosis_cmd.extend(['threads=1', 'map-start-zoom=8'])
	if tag_mapping is not None:
		osmosis_cmd.extend(['tag-conf-file=' + os.path.abspath(tag_mapping)])
	cmd = subprocess.Popen(osmosis_cmd, env=env)
	cmd.wait()

def osmosis_pois(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space):
	output_dir = current_timestamp + '-' + output_map_name

	map_list = []
	for alist in map_lists:
		with open(alist, 'r') as maps:
			for line in maps:
				map_list.append(os.path.join(maps_dir, line.strip().split('/')[-1]+SOURCE_MAP_POSTFIX))

	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
	env = os.environ.copy()
	env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
	print('Processing POIs using osmosis')
	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
	for amap in map_list:
		osmosis_cmd.extend(['--rb', 'file=' + amap])
	for x in range(0, len(map_list) - 1):
		 osmosis_cmd.append('--merge')
	osmosis_cmd.extend(['--poi-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')])
	cmd = subprocess.Popen(osmosis_cmd, env=env)
	cmd.wait()