kemonine/osm_map_processing
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Misc updates/fixes (see below) - Added -server to JVM options to help increase performance during long-running creations (per osmosis perf tuning) - Improved command help text and usage info - Adjusted how wget/bunzip2 are found (improves reliability in finding them) - Added option to NOT sleep between downloads (use this at your own risk, mirrors WILL throttle generally) - Added option to use RAM instead of HD for processing (mapsforge eats RAM, be careful with this)

Browse source
This commit is contained in:
Mike C 2017-01-20 12:29:54 -05:00
parent 25d495e62f
commit c67b371856
1 changed files with 17 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
process_maps.py
View file

@ -19,14 +19,19 @@ env = os.environ.copy()
FNULL = open(os.devnull, 'w') FNULL = open(os.devnull, 'w')
wget_cmd = '/usr/bin/wget' wget_cmd = 'wget'
bunzip2_cmd = '/usr/bin/bunzip2' bunzip2_cmd = 'bunzip2'
if __name__ == '__main__': if __name__ == '__main__':
current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M') current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--map-list', action='append') parser.add_argument('--map-list', action='append',
help='a text file with one map URL per line, can be specified more than once')
parser.add_argument('--no-sleep', action='store_true',
help='don\'t sleep between downloads -- WARNING you can easily run into throttling on mirrors if you use this option')
parser.add_argument('--use-ram', action='store_true',
help='use RAM for mapsforge processing -- WARNING mapsforge uses 10x the map size in RAM for processing (ie. 100Mb map = 1Gb RAM usage), you want a LOT of RAM for this option')
#TODO: Add argument to pass path to osmosis #TODO: Add argument to pass path to osmosis
#TODO: Add argument to pass path to output dir #TODO: Add argument to pass path to output dir
args = parser.parse_args() args = parser.parse_args()
@ -54,8 +59,9 @@ if __name__ == '__main__':
print(' ', end='') print(' ', end='')
print(line) print(line)
subprocess.run([wget_cmd, '-P', 'dl', line.strip()], stdout=FNULL, stderr=subprocess.STDOUT) subprocess.run([wget_cmd, '-P', 'dl', line.strip()], stdout=FNULL, stderr=subprocess.STDOUT)
print(' Sleeping to prevent throttle/blocking') if not args.no_sleep:
time.sleep(300) # Seconds print(' Sleeping to prevent throttle/blocking')
time.sleep(300) # Seconds
print('Decompressing maps (if necessary)') print('Decompressing maps (if necessary)')
for dirpath, dirnames, filenames in os.walk('dl'): for dirpath, dirnames, filenames in os.walk('dl'):
@ -66,7 +72,7 @@ if __name__ == '__main__':
subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)]) subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)])
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) # Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
env['JAVACMD_OPTIONS'] = '-Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) env['JAVACMD_OPTIONS'] = '-server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
files_to_process = [] files_to_process = []
for dirpath, dirnames, filenames in os.walk('dl'): for dirpath, dirnames, filenames in os.walk('dl'):
for file in filenames: for file in filenames:
@ -83,7 +89,11 @@ if __name__ == '__main__':
osmosis_cmd.extend(['--rb', 'file=' + file]) osmosis_cmd.extend(['--rb', 'file=' + file])
for x in range(0, len(files_to_process) - 1): for x in range(0, len(files_to_process) - 1):
osmosis_cmd.append('--merge') osmosis_cmd.append('--merge')
osmosis_cmd.extend(['--mapfile-writer', 'file=output.map', 'type=hd']) osmosis_cmd.extend(['--mapfile-writer', 'file=output.map'])
if args.use_ram:
osmosis_cmd.extend(['type=ram'])
else:
osmosis_cmd.extend(['type=hd'])
cmd = subprocess.Popen(osmosis_cmd, env=env) cmd = subprocess.Popen(osmosis_cmd, env=env)
cmd.wait() cmd.wait()