Rework lists, update to match main mapsforge docs (land/sea polys in particular), overhaul README for better examples

3 years ago · 772f05f199
54 changed files with 952 additions and 397 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,6 +1,12 @@
+__pycache__
 out/
+out
 test/
+test
 tmp/
-*.swp
 tmp
+cache/
 cache
+logs/
+logs
+*.swp
--- a/README.md
+++ b/README.md
@ -10,7 +10,8 @@ To run this program you'll need the following
 * Python 3.x installed
 * Java 1.8 or higher installed (OpenJDK works)
 * wget installed
-* bunzip2 installed (usually part of bzip2 package)
+* gdal-bin
+* python3-gdal (Ubuntu) installed

 # Installation

@ -23,47 +24,52 @@ To run the program, cd to the directory where you extracted the release and run
 ### Program Usage

 ```
-usage: process_maps.py [-h] [--map-list MAP_LIST] [--no-sleep] [--use-ram]
+usage: process_maps.py [-h] --map-list MAP_LIST [--use-ram]
                       [--max-heap-space MAX_HEAP_SPACE]
-                       [--output-map-name OUTPUT_MAP_NAME]
-                       [--cached-maps-dir CACHED_MAPS_DIR]
+                       [--output-map-name OUTPUT_MAP_NAME] --maps-dir MAPS_DIR
+                       [--no-map-download]

 optional arguments:
  -h, --help            show this help message and exit
  --map-list MAP_LIST   a text file with one map URL per line, can be
                        specified more than once
-  --no-sleep            don't sleep between downloads -- WARNING you can
-                        easily run into throttling on mirrors if you use this
-                        option
  --use-ram             use RAM for mapsforge processing -- WARNING mapsforge
                        uses 10x the map size in RAM for processing (ie. 100Mb
                        map = 1Gb RAM usage), you want a LOT of RAM for this
                        option
  --max-heap-space MAX_HEAP_SPACE
                        set the max heap space for the JVM, use standard -Xmx
-                        values, default (4g) should be fine if not using
+                        values, default (1g) should be fine if not using
                        --use-ram argument
  --output-map-name OUTPUT_MAP_NAME
                        set the output .map and .poi file names
-  --cached-maps-dir CACHED_MAPS_DIR
-                        Use cached maps in the specified directory instead of
-                        downloads using map lists
+  --maps-dir MAPS_DIR   Where downloaded maps will be stored/read from
+  --no-map-download     Do NOT download maps, re-use maps from maps-dir
 ```

 ### Examples

 ```
-./process_maps.py --max-heap-space 24g \
-    --output-map-name central_america \
-    --cached-maps-dir ~/osmmapdata/cache/central_america/20180225-1429
-
-./process_maps.py --max-heap-space 24g \
-    --output-map-name midwest \
-    --cached-maps-dir ~/osmmapdata/cache/midwest/20180225-1429
+#./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt
+#./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt
+./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name netherlands --map-list lists/netherlands.txt
+./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name spain --map-list lists/spain.txt
+./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name france --map-list lists/france.txt
+./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia_oceania --map-list lists/australia_oceania.txt
+./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia --map-list lists/australia.txt
+./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name canada_ontario --map-list lists/ontario.txt
+./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name local --map-list lists/midwest.txt --map-list lists/ontario.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_midwest --map-list lists/usa_midwest.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_northeast --map-list lists/usa_northeast.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_pacific --map-list lists/usa_pacific.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_south --map-list lists/usa_south.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_west --map-list lists/usa_west.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa --map-list lists/usa.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name canada --map-list lists/canada.txt
+./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name north_america --map-list lists/north_america.txt

-./process_maps.py --max-heap-space 24g \
-    --output-map-name south_america \
-    --cached-maps-dir ~/osmmapdata/cache/south_america/20180225-1429
 ```

 # Caching maps for re-use / re-processing
@ -73,37 +79,27 @@ Included in the repo there is a ```download_maps.py``` script that is meant to h
 ### Program usage

 ```
-usage: download_maps.py [-h] [--map-list MAP_LIST] [--no-sleep]
-                        [--output-map-name OUTPUT_MAP_NAME]
-                        [--cached-maps-dir CACHED_MAPS_DIR]
+usage: download_maps.py [-h] --map-list MAP_LIST --map-dir MAP_DIR

 optional arguments:
-  -h, --help            show this help message and exit
-  --map-list MAP_LIST   a text file with one map URL per line, can be
-                        specified more than once
-  --no-sleep            don't sleep between downloads -- WARNING you can
-                        easily run into throttling on mirrors if you use this
-                        option
-  --output-map-name OUTPUT_MAP_NAME
-                        set the name of the map directory before Ymd-HM
-  --cached-maps-dir CACHED_MAPS_DIR
-                        The root directory where maps should be cached
+  -h, --help           show this help message and exit
+  --map-list MAP_LIST  a text file with one map URL per line, can be specified
+                       more than once
+  --map-dir MAP_DIR    The directory where maps should be downloaded
+
 ```

 ### Example Usage

 ```
-./download_maps.py --cached-maps-dir ./cache \
-    --output-map-name central_america \
-    --map-list lists/pbf/central_america.txt
+./download_maps.py --map-dir ./cache \
+    --map-list lists/central_america.txt

-./download_maps.py --cached-maps-dir ./cache \
-    --output-map-name midwest \
-    --map-list lists/pbf/midwest.txt
+./download_maps.py --map-dir ./cache \
+    --map-list lists/midwest.txt

-./download_maps.py --cached-maps-dir ./cache \
-    --output-map-name south_america \
-    --map-list lists/pbf/south_america.txt
+./download_maps.py --map-dir ./cache \
+    --map-list lists/south_america.txt
 ```

 # Licencing
--- a/azure_notes.md
+++ b/azure_notes.md
@ -82,22 +82,19 @@ Download maps using lists to local cache (use a blob storage endpoint, these can

 ```
 # Example, reformat to your needs/desires
-./download_maps.py --map-list lists/test.txt --cached-maps-dir ./cache --output-map-name test
+./download_maps.py --map-list lists/test.txt --map-dir ./cache
 ```

 ### Midwest USA

 ```
-./process_maps.py --use-ram --max-heap-space 48g \
-    --output-map-name midwest \
-    --cached-maps-dir ~/osmmapdata/cache/midwest/20180225-1429
+./process_maps.py --use-ram --max-heap-space 50g --maps-dir cache --no-map-download --output-map-name midwest --map-list lists/midwest.txt --map-list lists/ontario.txt
 ```

 ### South America
 ```
-./process_maps.py --use-ram --max-heap-space 48g \
-    --output-map-name south_america \
-    --cached-maps-dir ~/osmmapdata/cache/south_america/20180225-1429
+
+./process_maps.py --use-ram --max-heap-space 50g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt
 ```

 ### Central America
@ -105,8 +102,6 @@ Download maps using lists to local cache (use a blob storage endpoint, these can
 *NOTE: This CAN NOT fit into 64Gb RAM, you'll need to setup a 128Gb SSD/Disk at ~/osm_map_processing/tmp for this to not enounter heap space errors* 

 ```
-./process_maps.py --max-heap-space 48g \
-    --output-map-name central_america \
-    --cached-maps-dir ~/osmmapdata/cache/central_america/20180225-1429
+./process_maps.py --max-heap-space 50g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt
 ```

--- a/download_maps.py
+++ b/download_maps.py
@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-

-# Copyright 2016 Mike "KemoNine" Crosson
+# Copyright 2018 Mike "KemoNine" Crosson
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@ -12,67 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-import subprocess, sys, os, pprint, datetime, argparse, time
-
-base_path = os.path.dirname(os.path.realpath(__file__))
-env = os.environ.copy()
-
-FNULL = open(os.devnull, 'w')
-
-wget_cmd = 'wget'
-bunzip2_cmd = 'bunzip2'
+import argparse
+from pymapsforge import download

 if __name__ == '__main__':
-	current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
-	
 	parser = argparse.ArgumentParser()
-	parser.add_argument('--map-list', action='append',
+	parser.add_argument('--map-list', action='append', required=True,
                        help='a text file with one map URL per line, can be specified more than once')
-	parser.add_argument('--no-sleep', action='store_true',
-                        help='don\'t sleep between downloads -- WARNING you can easily run into throttling on mirrors if you use this option')
-	parser.add_argument('--output-map-name', action='store', default='output',
-                        help='set the name of the map directory before Ymd-HM')
-	parser.add_argument('--cached-maps-dir', action='store',
-                        help='The root directory where maps should be cached')
+	parser.add_argument('--map-dir', action='store', required=True,
+                        help='The directory where maps should be downloaded')
 	args = parser.parse_args()

-	if args.map_list is None or args.cached_maps_dir is None:
-		print('You MUST specify at least one map-list AND cached-maps-dir')
-		sys.exit(1)
-
-	# Normalize map path directory based on CLI arg ahead of any path manipulations
-	map_dl_dirs = [args.cached_maps_dir]
-	if args.output_map_name is not None:
-		map_dl_dirs.append(args.output_map_name)
-	map_dl_dirs.append(current_timestamp)
-
-	cached_maps_dir = os.path.abspath(os.path.join(*map_dl_dirs))
-
-	print('Downloading maps to : ' + cached_maps_dir)
-
-	map_list = []
-
-	if args.map_list is not None:
-		for alist in args.map_list:
-			with open(alist, 'r') as maps:
-				for line in maps:
-					map_list.append(line.strip())
-
-	if args.map_list is not None:
-		print('Downloading maps')
-		for line in map_list:
-			print('    ', end='')
-			print(line)
-			subprocess.run([wget_cmd, '-P', cached_maps_dir, line.strip()], stdout=FNULL, stderr=subprocess.STDOUT)
-			if not args.no_sleep:
-				print('    Sleeping to prevent throttle/blocking')
-				time.sleep(300) # Seconds
-
-		print('Decompressing maps (if necessary)')
-		for dirpath, dirnames, filenames in os.walk(cached_maps_dir):
-			for file in filenames:
-				if file.endswith('bz2'):
-					print('    ', end='')
-					print(file)
-					subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)])
+	download.download(args.map_dir, args.map_list)

--- a/lists/australia.txt
+++ b/lists/australia.txt
@ -0,0 +1 @@
+australia-oceania/australia
--- a/lists/australia_oceania.txt
+++ b/lists/australia_oceania.txt
@ -0,0 +1 @@
+australia-oceania
--- a/lists/canada.txt
+++ b/lists/canada.txt
@ -0,0 +1 @@
+north-america/canada
--- a/lists/central_america.txt
+++ b/lists/central_america.txt
@ -0,0 +1,2 @@
+central-america
+north-america/mexico
--- a/lists/deleware.txt
+++ b/lists/deleware.txt
@ -0,0 +1 @@
+north-america/us/delaware
--- a/lists/france.txt
+++ b/lists/france.txt
@ -0,0 +1 @@
+europe/france
--- a/lists/idaho.txt
+++ b/lists/idaho.txt
@ -0,0 +1 @@
+north-america/us/idaho
--- a/lists/mexico.txt
+++ b/lists/mexico.txt
@ -0,0 +1 @@
+north-america/mexico
--- a/lists/michigan.txt
+++ b/lists/michigan.txt
@ -0,0 +1 @@
+north-america/us/michigan
--- a/lists/midwest.txt
+++ b/lists/midwest.txt
@ -0,0 +1,6 @@
+north-america/us/illinois
+north-america/us/indiana
+north-america/us/michigan
+north-america/us/minnesota
+north-america/us/ohio
+north-america/us/wisconsin
--- a/lists/netherlands.txt
+++ b/lists/netherlands.txt
@ -0,0 +1 @@
+europe/netherlands
--- a/lists/north_america.txt
+++ b/lists/north_america.txt
@ -0,0 +1 @@
+north-america
--- a/lists/ohio.txt
+++ b/lists/ohio.txt
@ -0,0 +1 @@
+north-america/us/ohio
--- a/lists/ontario.txt
+++ b/lists/ontario.txt
@ -0,0 +1 @@
+north-america/canada/ontario
--- a/lists/osm/canada.txt
+++ b/lists/osm/canada.txt
@ -1,13 +0,0 @@
-http://download.geofabrik.de/north-america/canada/alberta-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/british-columbia-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/manitoba-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/new-brunswick-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/newfoundland-and-labrador-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/northwest-territories-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/nova-scotia-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/nunavut-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/ontario-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/quebec-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/saskatchewan-latest.osm.bz2
-http://download.geofabrik.de/north-america/canada/yukon-latest.osm.bz2
--- a/lists/osm/mexico.txt
+++ b/lists/osm/mexico.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/mexico-latest.osm.bz2
--- a/lists/osm/midwest.txt
+++ b/lists/osm/midwest.txt
@ -1,6 +0,0 @@
-http://download.geofabrik.de/north-america/us/illinois-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/indiana-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/michigan-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/minnesota-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/ohio-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.bz2
--- a/lists/osm/ontario.txt
+++ b/lists/osm/ontario.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/canada/ontario-latest.osm.bz2
--- a/lists/osm/usa.txt
+++ b/lists/osm/usa.txt
@ -1,51 +0,0 @@
-http://download.geofabrik.de/north-america/us/alabama-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/alaska-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/arizona-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/arkansas-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/california-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/colorado-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/connecticut-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/delaware-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/florida-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/georgia-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/hawaii-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/idaho-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/illinois-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/indiana-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/iowa-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/kansas-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/kentucky-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/louisiana-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/maine-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/maryland-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/michigan-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/minnesota-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/mississippi-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/missouri-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/montana-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/nebraska-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/nevada-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/new-hampshire-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/new-jersey-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/new-mexico-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/new-york-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/north-carolina-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/north-dakota-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/ohio-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/oklahoma-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/oregon-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/rhode-island-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/south-carolina-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/south-dakota-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/tennessee-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/texas-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/utah-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/vermont-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/virginia-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/washington-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/west-virginia-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.bz2
-http://download.geofabrik.de/north-america/us/wyoming-latest.osm.bz2
--- a/lists/pbf/australia_oceania.txt
+++ b/lists/pbf/australia_oceania.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/australia-oceania-latest.osm.pbf
--- a/lists/pbf/canada.txt
+++ b/lists/pbf/canada.txt
@ -1,13 +0,0 @@
-http://download.geofabrik.de/north-america/canada/alberta-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/british-columbia-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/manitoba-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/new-brunswick-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/newfoundland-and-labrador-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/northwest-territories-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/nova-scotia-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/nunavut-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/ontario-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/quebec-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/saskatchewan-latest.osm.pbf
-http://download.geofabrik.de/north-america/canada/yukon-latest.osm.pbf
--- a/lists/pbf/central_america.txt
+++ b/lists/pbf/central_america.txt
@ -1,2 +0,0 @@
-http://download.openstreetmap.fr/extracts/central-america-latest.osm.pbf
-http://download.geofabrik.de/north-america/mexico-latest.osm.pbf
--- a/lists/pbf/idaho.txt
+++ b/lists/pbf/idaho.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/us/idaho-latest.osm.pbf
--- a/lists/pbf/mexico.txt
+++ b/lists/pbf/mexico.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/mexico-latest.osm.pbf
--- a/lists/pbf/michigan.txt
+++ b/lists/pbf/michigan.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf
--- a/lists/pbf/midwest.txt
+++ b/lists/pbf/midwest.txt
@ -1,6 +0,0 @@
-http://download.geofabrik.de/north-america/us/illinois-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/indiana-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/minnesota-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/ohio-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.pbf
--- a/lists/pbf/ontario.txt
+++ b/lists/pbf/ontario.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/canada/ontario-latest.osm.pbf
--- a/lists/pbf/pennsylvania.txt
+++ b/lists/pbf/pennsylvania.txt
@ -1 +0,0 @@
-http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.pbf
--- a/lists/pbf/south_america.txt
+++ b/lists/pbf/south_america.txt
@ -1 +0,0 @@
-http://download.openstreetmap.fr/extracts/south-america-latest.osm.pbf
--- a/lists/pbf/usa.txt
+++ b/lists/pbf/usa.txt
@ -1,51 +0,0 @@
-http://download.geofabrik.de/north-america/us/alabama-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/alaska-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/arizona-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/arkansas-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/california-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/colorado-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/connecticut-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/florida-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/georgia-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/hawaii-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/idaho-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/illinois-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/indiana-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/iowa-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/kansas-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/kentucky-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/louisiana-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/maryland-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/minnesota-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/mississippi-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/missouri-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/montana-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/nebraska-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/nevada-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/new-hampshire-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/new-jersey-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/new-mexico-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/new-york-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/north-carolina-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/north-dakota-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/ohio-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/oklahoma-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/oregon-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/south-carolina-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/south-dakota-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/tennessee-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/texas-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/utah-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/vermont-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/virginia-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/washington-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/west-virginia-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/wyoming-latest.osm.pbf
--- a/lists/pennsylvania.txt
+++ b/lists/pennsylvania.txt
@ -0,0 +1 @@
+north-america/us/pennsylvania
--- a/lists/south_america.txt
+++ b/lists/south_america.txt
@ -0,0 +1 @@
+south-america
--- a/lists/spain.txt
+++ b/lists/spain.txt
@ -0,0 +1 @@
+europe/spain
--- a/lists/test.txt
+++ b/lists/test.txt
@ -1,2 +1,2 @@
-http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.pbf
-http://download.geofabrik.de/north-america/us/delaware-latest.osm.bz2
+north-america/canada/prince-edward-island
+north-america/us/delaware
--- a/lists/usa.txt
+++ b/lists/usa.txt
@ -0,0 +1,51 @@
+north-america/us/alabama
+north-america/us/alaska
+north-america/us/arizona
+north-america/us/arkansas
+north-america/us/california
+north-america/us/colorado
+north-america/us/connecticut
+north-america/us/delaware
+north-america/us/district-of-columbia
+north-america/us/florida
+north-america/us/georgia
+north-america/us/hawaii
+north-america/us/idaho
+north-america/us/illinois
+north-america/us/indiana
+north-america/us/iowa
+north-america/us/kansas
+north-america/us/kentucky
+north-america/us/louisiana
+north-america/us/maine
+north-america/us/maryland
+north-america/us/massachusetts
+north-america/us/michigan
+north-america/us/minnesota
+north-america/us/mississippi
+north-america/us/missouri
+north-america/us/montana
+north-america/us/nebraska
+north-america/us/nevada
+north-america/us/new-hampshire
+north-america/us/new-jersey
+north-america/us/new-mexico
+north-america/us/new-york
+north-america/us/north-carolina
+north-america/us/north-dakota
+north-america/us/ohio
+north-america/us/oklahoma
+north-america/us/oregon
+north-america/us/pennsylvania
+north-america/us/rhode-island
+north-america/us/south-carolina
+north-america/us/south-dakota
+north-america/us/tennessee
+north-america/us/texas
+north-america/us/utah
+north-america/us/vermont
+north-america/us/virginia
+north-america/us/washington
+north-america/us/west-virginia
+north-america/us/wisconsin
+north-america/us/wyoming
--- a/lists/usa_midwest.txt
+++ b/lists/usa_midwest.txt
@ -0,0 +1 @@
+north-america/us-midwest
--- a/lists/usa_northeast.txt
+++ b/lists/usa_northeast.txt
@ -0,0 +1 @@
+north-america/us-northeast
--- a/lists/usa_pacific.txt
+++ b/lists/usa_pacific.txt
@ -0,0 +1 @@
+north-america/us-pacific
--- a/lists/usa_south.txt
+++ b/lists/usa_south.txt
@ -0,0 +1 @@
+north-america/us-south
--- a/lists/usa_west.txt
+++ b/lists/usa_west.txt
@ -0,0 +1 @@
+north-america/us-west
--- a/manual_processing.md
+++ b/manual_processing.md
@ -1,25 +0,0 @@
-Manual Procedure
-=
-
-Prep 
-== 
- Download updated maps: http://download.geofabrik.de/north-america.html and http://download.geofabrik.de/north-america/canada.html  
- Pull latest osmosis from http://wiki.openstreetmap.org/wiki/Osmosis 
- Pull latest mapsforge for osmosis from https://github.com/mapsforge/mapsforge/blob/master/docs/Downloads.md 
- 
-Process maps for device 
-== 
- Setup osmosis in a directory (unzip release -- ZIPBOMB ALERT!!!!) 
- Create plugins directory under osmosis 
- Move mapsforge release jar to osmosis/lib/default/ 
- Decompress maps (if they were previously compressed) 
- ./osmosis bin/osmosis --rb file=../maps/a_map.osm.pbf --mapfile-writer file=../maps/a_map.map 
-  * If you run out of heap space add JAVACMD_OPTIONS="-Xmx16g" to the start of the line 
-  * If increasing the heap doesn't work, append type=hd to end of command 
-  * You can merge maps using --merge passed to osmosis (before --mapfile-writer) using n-1 instances of --merge where n is the number of --rb or --rx options passed 
- 
-General Notes 
-= 
-Used to generate above info 
-* https://github.com/mapsforge/mapsforge/blob/master/docs/Getting-Started-Map-Writer.md
-
--- a/process_all.sh
+++ b/process_all.sh
@ -0,0 +1,13 @@
+#!/bin/bash
+
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name local --map-list lists/michigan.txt --map-list lists/ontario.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name midwest --map-list lists/midwest.txt --map-list lists/ontario.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia_oceania --map-list lists/australia_oceania.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name usa --map-list lists/usa.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name canada --map-list lists/canada.txt
+./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name north_america --map-list lists/usa.txt --map-list lists/canada.txt
+
--- a/process_maps.py
+++ b/process_maps.py
@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-

-# Copyright 2016 Mike "KemoNine" Crosson
+# Copyright 2018 Mike "KemoNine" Crosson
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@ -12,124 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-import subprocess, sys, os, pprint, datetime, argparse, time
-
-base_path = os.path.dirname(os.path.realpath(__file__))
-env = os.environ.copy()
-
-FNULL = open(os.devnull, 'w')
-
-wget_cmd = 'wget'
-bunzip2_cmd = 'bunzip2'
+import argparse, datetime, os
+from pymapsforge import process, download

 if __name__ == '__main__':
 	current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
 	
 	parser = argparse.ArgumentParser()
-	parser.add_argument('--map-list', action='append',
+	parser.add_argument('--map-list', action='append', required=True,
                        help='a text file with one map URL per line, can be specified more than once')
-	parser.add_argument('--no-sleep', action='store_true',
-                        help='don\'t sleep between downloads -- WARNING you can easily run into throttling on mirrors if you use this option')
 	parser.add_argument('--use-ram', action='store_true',
                        help='use RAM for mapsforge processing -- WARNING mapsforge uses 10x the map size in RAM for processing (ie. 100Mb map = 1Gb RAM usage), you want a LOT of RAM for this option')
-	parser.add_argument('--max-heap-space', action='store', default='4g',
-                        help='set the max heap space for the JVM, use standard -Xmx values, default (4g) should be fine if not using --use-ram argument')
+	parser.add_argument('--max-heap-space', action='store', default='1g',
+                        help='set the max heap space for the JVM, use standard -Xmx values, default (1g) should be fine if not using --use-ram argument')
 	parser.add_argument('--output-map-name', action='store', default='output',
                        help='set the output .map and .poi file names')
-	parser.add_argument('--cached-maps-dir', action='store',
-                        help='Use cached maps in the specified directory instead of downloads using map lists')
-	#TODO: Add argument to pass path to osmosis
-	#TODO: Add argument to pass path to output dir
+	parser.add_argument('--maps-dir', action='store', required=True,
+                        help='Where downloaded maps will be stored/read from')
+	parser.add_argument('--no-map-download', action='store_true',
+                        help='Do NOT download maps, re-use maps from maps-dir')
 	args = parser.parse_args()

-	if args.map_list is None and args.cached_maps_dir is None:
-		print('You MUST specify at least one map-list or cached-maps-dir')
-		sys.exit(1)
-
-	# Normalize map path directory based on CLI arg ahead of any path manipulations
-	# DEFAULTS TO 'dl' (non-normalized) to retain original behavior of script
-	cached_maps_dir = 'dl'
-	if args.cached_maps_dir is not None:
-		cached_maps_dir = os.path.abspath(args.cached_maps_dir)
-
-	map_list = []
-
-	if args.map_list is not None:
-		for alist in args.map_list:
-			with open(alist, 'r') as maps:
-				for line in maps:
-					map_list.append(line.strip())
-
-	print('Creating working directories')
-	if not os.path.exists('tmp'):
-		os.makedirs('tmp')
-	if not os.path.exists('out'):
-		os.makedirs('out')
-	os.chdir('out')
-	output_dir = current_timestamp + '-' + args.output_map_name
-	if not os.path.exists(output_dir):
-		os.makedirs(output_dir)
-	os.chdir(output_dir)
-
-	if args.map_list is not None:
-		if not os.path.exists('dl'):
-			os.makedirs('dl')
-
-		print('Downloading maps')
-		for line in map_list:
-			print('    ', end='')
-			print(line)
-			subprocess.run([wget_cmd, '-P', 'dl', line.strip()], stdout=FNULL, stderr=subprocess.STDOUT)
-			if not args.no_sleep:
-				print('    Sleeping to prevent throttle/blocking')
-				time.sleep(300) # Seconds
-
-		print('Decompressing maps (if necessary)')
-		for dirpath, dirnames, filenames in os.walk('dl'):
-			for file in filenames:
-				if file.endswith('bz2'):
-					print('    ', end='')
-					print(file)
-					subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)])
-				
-	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
-	env['JAVACMD_OPTIONS'] = '-Xmx' + args.max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
-	print('Finding maps to process')
-	files_to_process = []
-	for dirpath, dirnames, filenames in os.walk(cached_maps_dir):
-		for file in filenames:
-			print('    Found map: ', end='')
-			print(os.path.join(dirpath, file))
-			files_to_process.append(os.path.join(dirpath, file))
+	current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
+	base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__))))
+	args.maps_dir = os.path.abspath(args.maps_dir)
 	
-	print('Processing maps using osmosis')
-	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
-	for file in files_to_process:
-		if file.endswith('osm'):
-			osmosis_cmd.extend(['--rx', 'file=' + file])
-		elif file.endswith('pbf'):
-			osmosis_cmd.extend(['--rb', 'file=' + file])
-	for x in range(0, len(files_to_process) - 1):
-		osmosis_cmd.append('--merge')
-	osmosis_cmd.extend(['--mapfile-writer', 'file=' + args.output_map_name + '.map'])
-	if args.use_ram:
-		osmosis_cmd.extend(['type=ram'])
-	else:
-		osmosis_cmd.extend(['type=hd'])
-	osmosis_cmd.extend(['threads=4'])
-	cmd = subprocess.Popen(osmosis_cmd, env=env)
-	cmd.wait()
+	process.setup_working_dirs(base_path, current_timestamp, args.output_map_name)
+
+	if not args.no_map_download:
+		download.download(args.maps_dir, args.map_list)

-	print('Processing POIs using osmosis')
-	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
-	for file in files_to_process:
-		if file.endswith('osm'):
-			osmosis_cmd.extend(['--rx', 'file=' + file])
-		elif file.endswith('pbf'):
-			osmosis_cmd.extend(['--rb', 'file=' + file])
-	for x in range(0, len(files_to_process) - 1):
-		 osmosis_cmd.append('--merge')
-	osmosis_cmd.extend(['--poi-writer', 'file=' + args.output_map_name + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')])
-	cmd = subprocess.Popen(osmosis_cmd, env=env)
-	cmd.wait()
+	process.prep_land_sea_polys(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name)
+	process.process_land_sea_polys(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name, args.max_heap_space, args.use_ram)
+	process.osmosis_main(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name, args.max_heap_space, args.use_ram)
+	process.osmosis_pois(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name, args.max_heap_space)

--- a/pymapsforge/init.py
+++ b/pymapsforge/init.py
--- a/pymapsforge/download.py
+++ b/pymapsforge/download.py
@ -0,0 +1,63 @@
+# Copyright 2018 Mike "KemoNine" Crosson
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess, sys, os, pprint, datetime, argparse, time
+from . import md5_map
+
+MIRROR_BASE_URL = 'http://download.geofabrik.de/'
+MIRROR_FILE_SUFFIX = '-latest.osm.pbf'
+MIRROR_MD5_SUFFIX = MIRROR_FILE_SUFFIX + '.md5'
+MIRROR_POLY_SUFFIX = '.poly'
+
+def download(dl_dir, dl_lists):
+
+	FNULL = open(os.devnull, 'w')
+
+	wget_cmd = 'wget'
+	bunzip2_cmd = 'bunzip2'
+
+	cached_maps_dir = os.path.abspath(dl_dir)
+
+	print('Downloading maps to : ' + cached_maps_dir)
+
+	map_list = []
+
+	for alist in dl_lists:
+		with open(alist, 'r') as maps:
+			for line in maps:
+				map_list.append(line.strip())
+
+	print('Downloading land polygons from data.openstreetmapdata.com')
+	subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, 'http://data.openstreetmapdata.com/land-polygons-split-4326.zip'], stdout=FNULL, stderr=subprocess.STDOUT)
+
+	print('Downloading maps (.osm.pbf, .md5, .poly)')
+	first_iteration = True
+	for amap in map_list:
+		if not first_iteration:
+			print('    Sleeping to prevent throttle/blocking')
+			time.sleep(30) # Seconds
+		else:
+			first_iteration = False
+		print('    ', end='')
+		print(amap)
+		subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_FILE_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
+		subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_MD5_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
+		subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_POLY_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
+
+	print('Checking map md5s')
+	for amap in map_list:
+		map_root = amap.strip().split('/')[-1]
+		map_file = os.path.join(cached_maps_dir, map_root + MIRROR_FILE_SUFFIX)
+		md5_file = os.path.join(cached_maps_dir, map_root + MIRROR_MD5_SUFFIX)
+		print('    ', end='')
+		print(amap + ': ', end='')
+		print(md5_map.check_hash(map_file, md5_file))
+
--- a/pymapsforge/md5_map.py
+++ b/pymapsforge/md5_map.py
@ -0,0 +1,32 @@
+# Copyright 2018 Mike "KemoNine" Crosson
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Borrowed from the web: https://www.joelverhagen.com/blog/2011/02/md5-hash-of-file-in-python/
+
+import hashlib
+
+def md5Checksum(filePath):
+	with open(filePath, 'rb') as fh:
+		m = hashlib.md5()
+		while True:
+			data = fh.read(8192)
+			if not data:
+				break
+			m.update(data)
+		return m.hexdigest()
+
+def check_hash(map_file, md5_file):
+	published_md5 = None
+	with open(md5_file, 'r') as f:
+	    published_md5 = f.readline().strip().split()[0]
+	map_md5 = md5Checksum(map_file)
+	return published_md5 == map_md5
+
--- a/pymapsforge/poly.py
+++ b/pymapsforge/poly.py
@ -0,0 +1,92 @@
+# Copyright 2018 Mike "KemoNine" Crosson
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Adapted from https://github.com/mapsforge/mapsforge-creator/blob/master/poly2bb.pl
+
+import re, os
+
+BOUND_BUFFER = 0.1
+COORD_MATCHER = re.compile('^\s*([0-9.E+-]+)\s+([0-9.E+-]+)\s*$')
+
+def calculate_total_bounds(maps_dir, map_lists):
+	cached_maps_dir = os.path.abspath(maps_dir)
+	x_min = 360.0
+	x_max = -360.0
+	y_min = 360.0
+	y_max = -360.0
+	for alist in map_lists:
+		with open(alist, 'r') as maps:
+			for line in maps:
+				current_map = line.strip().split('/')[-1]
+				bounds = calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly'))
+				if bounds['bottom'] < y_min:
+					y_min = bounds['bottom']
+				if bounds['left'] < x_min:
+					x_min = bounds['left']
+				if bounds['top'] > y_max:
+					y_max = bounds['top']
+				if bounds['right'] > x_max:
+					x_max = bounds['right']
+	values = {}
+	values['bottom'] = y_min
+	values['left'] = x_min
+	values['top'] = y_max
+	values['right'] = x_max
+	return values
+
+def calculate_bounds(mappoly):
+	# Opposite of max possible values so we find the right bounding box later
+	x_min = 360.0
+	x_max = -360.0
+	y_min = 360.0
+	y_max = -360.0
+
+	with open(mappoly, 'r') as polyfile:
+		for line in polyfile:
+			if COORD_MATCHER.match(line.strip()):
+				values = line.split()
+				x = float(values[0])
+				y = float(values[1])
+				if x < x_min:
+					x_min = x
+				if x > x_max:
+					x_max = x
+				if y < y_min:
+					y_min = y
+				if y > y_max:
+					y_max = y
+
+	# Buffer bounds
+	x_min = x_min - BOUND_BUFFER
+	x_max = x_max + BOUND_BUFFER
+	y_min = y_min - BOUND_BUFFER
+	y_max = y_max + BOUND_BUFFER
+
+	# Cleanup bounding (per original script)
+	x_min = -180 if x_min < -180 else x_min
+	x_max = 180 if x_max > 180 else x_max
+	y_min = -90 if y_min < -90 else y_min
+	y_max = 90 if y_max > 90 else y_max
+
+	# Dict w/ useful values
+	values = {}
+	values['bottom'] = y_min
+	values['left'] = x_min
+	values['top'] = y_max
+	values['right'] = x_max
+
+	x_center = (x_min + x_max) / 2.0
+	y_center = (y_min + y_max) / 2.0
+
+	values['lat'] = y_center
+	values['lon'] = x_center
+
+	return values
--- a/pymapsforge/process.py
+++ b/pymapsforge/process.py
@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike "KemoNine" Crosson
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess, os, zipfile, shutil, fileinput, glob
+from . import poly
+
+FNULL = open(os.devnull, 'w')
+
+SOURCE_MAP_POSTFIX = '-latest.osm.pbf'
+
+def setup_working_dirs(base_path, current_timestamp, output_map_name):
+	print('Creating working directories as needed')
+	if not os.path.exists(os.path.join(base_path, 'tmp')):
+		os.makedirs(os.path.join(base_path, 'tmp'))
+	if not os.path.exists(os.path.join(base_path, 'out')):
+		os.makedirs(os.path.join(base_path, 'out'))
+	output_dir = current_timestamp + '-' + output_map_name
+	if not os.path.exists(os.path.join(base_path, 'out', output_dir)):
+		os.makedirs(os.path.join(base_path, 'out', output_dir))
+
+def prep_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name):
+	print('Prepping land/sea polygons (one prep per map in list(s))')
+	output_dir = current_timestamp + '-' + output_map_name
+	polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
+	polys_zip = zipfile.ZipFile(os.path.join(maps_dir, 'land-polygons-split-4326.zip'))
+	
+	for alist in map_lists:
+		with open(alist, 'r') as maps:
+			for line in maps:
+				map_polys_dir = os.path.join(polys_output_dir, line.strip().split('/')[-1])
+				print('    Extracting to: ' + map_polys_dir)
+				os.makedirs(map_polys_dir)
+				polys_zip.extractall(map_polys_dir)
+ 
+	polys_zip.close()
+
+def process_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram):
+	print('Processing land/sea polygons (one process per map in list(s))')
+	output_dir = current_timestamp + '-' + output_map_name
+	polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
+	cached_maps_dir = os.path.abspath(maps_dir)
+	for alist in map_lists:
+		with open(alist, 'r') as maps:
+			for line in maps:
+				current_map = line.strip().split('/')[-1]
+				print('    Processing: ' + current_map)
+				bounds = poly.calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly'))
+				subprocess.run(['ogr2ogr', '-overwrite', '-progress', '-skipfailures', '-clipsrc', str(bounds['left']), str(bounds['bottom']), str(bounds['right']), str(bounds['top']), os.path.join(polys_output_dir, current_map + '.shp'), os.path.join(polys_output_dir, current_map, 'land-polygons-split-4326', 'land_polygons.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
+				subprocess.run(['python2', os.path.join(base_path, 'shape2osm.py'), '-l', os.path.join(polys_output_dir, current_map), os.path.join(polys_output_dir, current_map + '.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
+				shutil.copyfile(os.path.join(base_path, 'sea.osm'),  os.path.join(polys_output_dir, current_map + '_sea.osm'))
+				with fileinput.FileInput(os.path.join(polys_output_dir, current_map + '_sea.osm'), inplace=True, backup='.bak') as sea:
+					for line in sea:
+						updated_line = line.replace('$BOTTOM', str(bounds['bottom']))
+						updated_line = updated_line.replace('$LEFT', str(bounds['left']))
+						updated_line = updated_line.replace('$TOP', str(bounds['top']))
+						updated_line = updated_line.replace('$RIGHT', str(bounds['right']))
+						print(updated_line, end='')
+
+	map_list = []
+	for alist in map_lists:
+		with open(alist, 'r') as maps:
+			for line in maps:
+				map_list.append(line.strip().split('/')[-1])
+
+	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
+	env = os.environ.copy()
+	env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
+
+	print('Processing land/sea polys using osmosis')
+	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
+	for amap in map_list:
+		osmosis_cmd.extend(['--rb', 'file=' + os.path.join(maps_dir, amap + SOURCE_MAP_POSTFIX)])
+		for apolymap in glob.glob(os.path.join(polys_output_dir, amap + '*.osm')):
+			print('    Found poly map : ' + apolymap)
+			osmosis_cmd.extend(['--rx', 'file=' + apolymap, '--s', '--m'])
+	osmosis_cmd.extend(['--log-progress'])
+	for x in range(0, len(map_list) - 1):
+		osmosis_cmd.append('--merge')
+	osmosis_cmd.extend(['--log-progress'])
+	osmosis_cmd.extend(['--wb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
+	osmosis_cmd.extend(['omitmetadata=true'])
+	cmd = subprocess.Popen(osmosis_cmd, env=env)
+	cmd.wait()
+
+def osmosis_main(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram):
+	output_dir = current_timestamp + '-' + output_map_name
+
+	bounds = poly.calculate_total_bounds(maps_dir, map_lists)
+
+	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
+	env = os.environ.copy()
+	env['JAVACMD_OPTIONS'] = '-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
+	
+	print('-----------------------------')
+	print('Processing maps using osmosis')
+	print('-----------------------------')
+	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
+	osmosis_cmd.extend(['--rb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
+	osmosis_cmd.extend(['--log-progress'])
+	osmosis_cmd.extend(['--mapfile-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.map'])
+	if use_ram:
+		osmosis_cmd.extend(['type=ram'])
+	else:
+		osmosis_cmd.extend(['type=hd'])
+	osmosis_cmd.extend(['bbox=' + str(bounds['bottom']) + ',' + str(bounds['left']) + ',' + str(bounds['top']) + ',' + str(bounds['right'])])
+	osmosis_cmd.extend(['threads=1', 'map-start-zoom=8'])
+	cmd = subprocess.Popen(osmosis_cmd, env=env)
+	cmd.wait()
+
+def osmosis_pois(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space):
+	output_dir = current_timestamp + '-' + output_map_name
+
+	map_list = []
+	for alist in map_lists:
+		with open(alist, 'r') as maps:
+			for line in maps:
+				map_list.append(os.path.join(maps_dir, line.strip().split('/')[-1]+SOURCE_MAP_POSTFIX))
+
+	# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
+	env = os.environ.copy()
+	env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
+	print('Processing POIs using osmosis')
+	osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
+	for amap in map_list:
+		osmosis_cmd.extend(['--rb', 'file=' + amap])
+	for x in range(0, len(map_list) - 1):
+		 osmosis_cmd.append('--merge')
+	osmosis_cmd.extend(['--poi-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')])
+	cmd = subprocess.Popen(osmosis_cmd, env=env)
+	cmd.wait()
--- a/sea.osm
+++ b/sea.osm
@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<osm version="0.6">
+	<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459320" version="1" lon="$LEFT" lat="$BOTTOM" />
+	<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459321" version="1" lon="$LEFT" lat="$TOP" />
+	<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459322" version="1" lon="$RIGHT" lat="$TOP" />
+	<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459323" version="1" lon="$RIGHT" lat="$BOTTOM" />
+	<way timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951623372" version="1">
+		<nd ref="32951459320" />
+		<nd ref="32951459321" />
+		<nd ref="32951459322" />
+		<nd ref="32951459323" />
+		<nd ref="32951459320" />
+		<tag k="area" v="yes" />
+		<tag k="layer" v="-5" />
+		<tag k="natural" v="sea" />
+	</way>
+</osm>
--- a/shape2osm.py
+++ b/shape2osm.py
@ -0,0 +1,434 @@
+#!/usr/bin/python
+
+"""
+This script is designed to act as assistance in converting shapefiles
+to OpenStreetMap data. This file is optimized and tested with MassGIS
+shapefiles, converted to EPSG:4326 before being passed to the script.
+You can perform this conversion with 
+
+   ogr2ogr -t_srs EPSG:4326 new_file.shp old_file.shp
+
+It is expected that you will modify the fixed_tags, tag_mapping, and
+boring_tags attributes of this script before running. You should read,
+or at least skim, the code up until it says:
+
+  DO NOT CHANGE AFTER THIS LINE.
+
+to accomodate your own data. 
+
+Modified to support a maximum waylength and multiple outer ways in a
+multipolygon relationship
+
+adapted by devemux86
+"""
+
+__author__ = "Christopher Schmidt <crschmidt@crschmidt.net>"
+__version__ = "$Id: polyshp2osm-WaterBody.py,v 1.3 2009/03/30 20:54:13 tbook Exp $"
+
+gdal_install = """
+Installing GDAL depends on your platform. Information is available at:
+   
+   http://trac.osgeo.org/gdal/wiki/DownloadingGdalBinaries
+
+For Debian-based systems:
+
+   apt-get install python-gdal
+
+will usually suffice. 
+"""
+
+# These tags are attached to all exterior ways. You can put any key/value pairs
+# in this dictionary. 
+
+fixed_tags = {
+  'natural': 'nosea',
+  'layer': '-5'
+}
+
+# Here are a number of functions: These functions define tag mappings. The API
+# For these functions is that they are passed the attributes from a feature,
+# and they return a list of two-tuples which match to key/value pairs.
+
+# The following Ftypes are not imported
+# 46100 - Submerged Stream
+ignoreField = "tile_x"
+ignoreValues = []
+
+# The following Ftypes are imported, but don't have any OSM tags attached:
+# 36100 - Playa - An area from which water evaporates
+
+def ftype(data):
+    """Type of body - From NHD Ftype"""  
+    natural = {
+        'LakePond': 'water',
+        'SwampMarsh': 'wetland'
+    }
+
+    if 'ftype' in data:
+        if data['ftype'] in natural:
+            return [('natural', natural[data['ftype']])]
+    return None   
+
+def fcode(data):
+    """For features where the ftype is not specific enough"""
+    landuse = {
+        '43600': 'reservoir', #Reservoir
+        '43601': 'reservoir', #Reservoir
+        '43602': 'reservoir', #Reservoir
+        '43603': 'reservoir', #Reservoir
+        '43604': 'reservoir', #Reservoir
+        '43605': 'reservoir', #Reservoir
+        '43606': 'reservoir', #Reservoir
+        '43607': 'reservoir', #Reservoir
+        '43609': 'reservoir', #Reservoir
+        '43610': 'reservoir', #Reservoir
+        '43611': 'reservoir', #Reservoir
+        '43612': 'reservoir', #Reservoir
+        '43614': 'reservoir', #Reservoir
+        '43615': 'reservoir', #Reservoir
+        '43616': 'reservoir', #Reservoir
+        '43617': 'reservoir', #Reservoir
+        '43618': 'reservoir', #Reservoir
+        '43619': 'reservoir', #Reservoir
+        '43620': 'reservoir' #Reservoir
+    }
+
+    manmade = {
+	'43613': 'reservoir_covered' #Covered Reservoir
+    }
+
+    leisure = {
+        '43608': 'swimming_pool' #Swimming Pool
+    }
+
+    if 'fcode' in data:
+        keys = []
+        if data['fcode'] in landuse:
+            keys.append( ('landuse', landuse[data['fcode']]) )
+        if data['fcode'] in manmade:
+            keys.append( ('man_made', manmade[data['fcode']]) )
+        if data['fcode'] in leisure:
+            keys.append( ('leisure', leisure[data['fcode']]) )
+        return keys
+    return None
+
+# The most important part of the code: define a set of key/value pairs
+# to iterate over to generate keys. This is a list of two-tuples: first
+# is a 'key', which is only used if the second value is a string. In
+# that case, it is a map of lowercased fieldnames to OSM tag names: so
+# fee_owner maps to 'owner' in the OSM output.
+
+# if the latter is callable (has a __call__; is a function), then that
+# method is called, passing in a dict of feature attributes with
+# lowercased key names. Those functions can then return a list of
+# two-tuples to be used as tags, or nothin' to skip the tags.  
+
+
+tag_mapping = [ 
+    ('ftype', ftype),
+    ('fcode', fcode),
+    ('gnis_name', 'name'),
+    ('gnis_id', 'gnis:feature_id'),
+    ('elevation', 'ele'),
+]    
+
+# These tags are not exported, even with the source data; this should be
+# used for tags which are usually calculated in a GIS. AREA and LEN are
+# common.
+
+boring_tags = [ 'error', 'tile_x', 'tile_y' ]
+
+# Namespace is used to prefix existing data attributes. If 'None', or 
+# '--no-source' is set, then source attributes are not exported, only
+# attributes in tag_mapping.
+
+namespace = "NHD"
+#namespace = None 
+
+# Uncomment the "DONT_RUN = False" line to get started. 
+
+#DONT_RUN = True
+DONT_RUN = False
+
+# Set the maximum length of a way (in nodes) before it is split into
+# shorter ways
+
+Max_Waylength = 1500000
+
+# =========== DO NOT CHANGE AFTER THIS LINE. ===========================
+# Below here is regular code, part of the file. This is not designed to
+# be modified by users.
+# ======================================================================
+
+import sys
+
+try:
+    try:
+        from osgeo import ogr
+    except ImportError:
+        import ogr
+except ImportError:
+    __doc__ += gdal_install 
+    if DONT_RUN:
+        print __doc__
+        sys.exit(2)
+    print "OGR Python Bindings not installed.\n%s" % gdal_install
+    sys.exit(1)
+
+def close_file():
+    """ Internal. Close an open file."""
+    global open_file
+    if not open_file.closed: 
+        open_file.write("</osm>")
+        open_file.close()
+
+def start_new_file():
+    """ Internal. Open a new file, closing existing file if neccesary."""
+    global open_file, file_counter
+    file_counter += 1
+    if open_file:
+        close_file()
+    open_file = open("%s%s.osm" % (file_name, file_counter), "w")
+    print >>open_file, "<osm version='0.6'>"
+
+def clean_attr(val):
+    """Internal. Hacky way to make attribute XML safe."""
+    val = str(val)
+    val = val.replace("&", "&amp;").replace("'", "&quot;").replace("<", "&lt;").replace(">", "&gt;").strip()
+    return val
+
+def add_ring_nodes(ring):
+    """Internal. Write the outer ring nodes."""
+    global open_file, id_counter
+    ringways = []
+    ids = []
+    firstnode = id_counter
+    if range(ring.GetPointCount() - 1) == 0 or ring.GetPointCount() == 0:
+        print >>sys.stderr, "Degenerate ring." 
+        return    
+    for count in range(ring.GetPointCount() - 1):
+        ids.append(id_counter)
+        print >>open_file, "<node timestamp='1969-12-31T23:59:59Z' changeset='-1' id='%s' version='1' lon='%s' lat='%s' />" % (id_counter, ring.GetX(count), ring.GetY(count))
+        id_counter += 1
+        if (count > 0) and ((count % (Max_Waylength - 1)) == 0):
+            ringways.append(ids)
+            ids = []
+	    ids.append(id_counter - 1)
+    ids.append(firstnode)
+    ringways.append(ids)
+    return ringways    
+
+def add_ring_way(ring): 
+    """Internal. write out the 'holes' in a polygon."""
+    global open_file, id_counter
+    ids = []
+    ringways = []
+    for count in range(ring.GetPointCount() - 1):
+        ids.append(id_counter)
+        print >>open_file, "<node timestamp='1969-12-31T23:59:59Z' changeset='-1' version='1' id='%s' lon='%s' lat='%s' />" % (id_counter, ring.GetX(count), ring.GetY(count))
+        id_counter += 1
+    if len(ids) == 0:
+        return None
+    print >>open_file, "<way timestamp='1969-12-31T23:59:59Z' changeset='-1' id='%s' version='1'>" % id_counter
+    way_id = id_counter
+    id_counter += 1
+    count = 0
+    for i in ids:
+        print >>open_file, "<nd ref='%s' />" % i
+        count += 1
+        if count >= Max_Waylength - 1:
+            count = 0
+            print >>open_file, "</way>"
+            ringways.append(way_id)
+            print >>open_file, "<way timestamp='1969-12-31T23:59:59Z' changeset='-1'id='%s' version='1'>" % id_counter
+            way_id = id_counter
+            id_counter += 1
+            print >>open_file, "<nd ref='%s' />" % i
+    print >>open_file, "<nd ref='%s' />" % ids[0]
+    print >>open_file, "</way>"
+    ringways.append(way_id)
+
+    return ringways
+
+open_file = None
+
+file_name = None 
+
+id_counter = 22951459320
+
+file_counter = 0
+counter = 0
+
+class AppError(Exception): pass
+
+def run(filename, slice_count=1, obj_count=100000000, output_location=None, no_source=False):
+    """Run the converter. Requires open_file, file_name, id_counter,
+    file_counter, counter to be defined in global space; not really a very good
+    singleton."""
+    global id_counter, file_counter, counter, file_name, open_file, namespace
+    
+    if no_source:
+        namespace=None
+
+    if output_location:
+       file_name = output_location
+
+    ds = ogr.Open(filename)
+    if not ds:
+        raise AppError("OGR Could not open the file %s" % filename)
+    l = ds.GetLayer(0)
+   
+    max_objs_per_file = obj_count 
+
+    extent = l.GetExtent()
+    if extent[0] < -180 or extent[0] > 180 or extent[2] < -90 or extent[2] > 90:
+        raise AppError("Extent does not look like degrees; are you sure it is? \n(%s, %s, %s, %s)" % (extent[0], extent[2], extent[1], extent[3]))  
+    slice_width = (extent[1] - extent[0]) / slice_count
+
+    seen = {}
+
+    print "Running %s slices with %s base filename against shapefile %s" % (
+            slice_count, file_name, filename)
+
+    for i in range(slice_count): 
+
+        l.ResetReading()
+        l.SetSpatialFilterRect(extent[0] + slice_width * i, extent[2], extent[0] + (slice_width * (i + 1)), extent[3])
+
+        start_new_file()
+        f = l.GetNextFeature()
+        
+        obj_counter = 0
+        last_obj_split = 0
+
+        while f:
+            start_id_counter = id_counter
+            if f.GetFID() in seen:
+                f = l.GetNextFeature()
+                continue
+            seen[f.GetFID()] = True             
+
+#            done = False
+#            while f.GetField(ignoreField) in ignoreValues:
+#                if l.GetNextFeature() == None:
+#                    done = True
+#                    break
+#                f = l.GetNextFeature()
+#            if done == True:
+#                break
+            
+            outerways = []
+	    innerways = []
+        
+            geom = f.GetGeometryRef()
+            ring = geom.GetGeometryRef(0)
+
+            objcount = ring.GetPointCount()
+            for i in range(1, geom.GetGeometryCount()):
+	        objcount += geom.GetGeometryRef(i).GetPointCount()
+	        objcount += 1
+
+            if (obj_counter - last_obj_split + objcount) > max_objs_per_file:
+