From c67b3718560cd68e1d16c682536d2f45fa2be40c Mon Sep 17 00:00:00 2001 From: Mike C Date: Fri, 20 Jan 2017 12:29:54 -0500 Subject: [PATCH] Misc updates/fixes (see below) - Added -server to JVM options to help increase performance during long-running creations (per osmosis perf tuning) - Improved command help text and usage info - Adjusted how wget/bunzip2 are found (improves reliability in finding them) - Added option to NOT sleep between downloads (use this at your own risk, mirrors WILL throttle generally) - Added option to use RAM instead of HD for processing (mapsforge eats RAM, be careful with this) --- process_maps.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/process_maps.py b/process_maps.py index 2d9ef2c..08da07c 100755 --- a/process_maps.py +++ b/process_maps.py @@ -19,14 +19,19 @@ env = os.environ.copy() FNULL = open(os.devnull, 'w') -wget_cmd = '/usr/bin/wget' -bunzip2_cmd = '/usr/bin/bunzip2' +wget_cmd = 'wget' +bunzip2_cmd = 'bunzip2' if __name__ == '__main__': current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M') parser = argparse.ArgumentParser() - parser.add_argument('--map-list', action='append') + parser.add_argument('--map-list', action='append', + help='a text file with one map URL per line, can be specified more than once') + parser.add_argument('--no-sleep', action='store_true', + help='don\'t sleep between downloads -- WARNING you can easily run into throttling on mirrors if you use this option') + parser.add_argument('--use-ram', action='store_true', + help='use RAM for mapsforge processing -- WARNING mapsforge uses 10x the map size in RAM for processing (ie. 100Mb map = 1Gb RAM usage), you want a LOT of RAM for this option') #TODO: Add argument to pass path to osmosis #TODO: Add argument to pass path to output dir args = parser.parse_args() @@ -54,8 +59,9 @@ if __name__ == '__main__': print(' ', end='') print(line) subprocess.run([wget_cmd, '-P', 'dl', line.strip()], stdout=FNULL, stderr=subprocess.STDOUT) - print(' Sleeping to prevent throttle/blocking') - time.sleep(300) # Seconds + if not args.no_sleep: + print(' Sleeping to prevent throttle/blocking') + time.sleep(300) # Seconds print('Decompressing maps (if necessary)') for dirpath, dirnames, filenames in os.walk('dl'): @@ -66,7 +72,7 @@ if __name__ == '__main__': subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)]) # Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) - env['JAVACMD_OPTIONS'] = '-Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) + env['JAVACMD_OPTIONS'] = '-server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) files_to_process = [] for dirpath, dirnames, filenames in os.walk('dl'): for file in filenames: @@ -83,7 +89,11 @@ if __name__ == '__main__': osmosis_cmd.extend(['--rb', 'file=' + file]) for x in range(0, len(files_to_process) - 1): osmosis_cmd.append('--merge') - osmosis_cmd.extend(['--mapfile-writer', 'file=output.map', 'type=hd']) + osmosis_cmd.extend(['--mapfile-writer', 'file=output.map']) + if args.use_ram: + osmosis_cmd.extend(['type=ram']) + else: + osmosis_cmd.extend(['type=hd']) cmd = subprocess.Popen(osmosis_cmd, env=env) cmd.wait()