Browse Source

Rework lists, update to match main mapsforge docs (land/sea polys in particular), overhaul README for better examples

master
Mike C 4 years ago
parent
commit
772f05f199
  1. 8
      .gitignore
  2. 82
      README.md
  3. 15
      azure_notes.md
  4. 65
      download_maps.py
  5. 1
      lists/australia.txt
  6. 1
      lists/australia_oceania.txt
  7. 1
      lists/canada.txt
  8. 2
      lists/central_america.txt
  9. 1
      lists/deleware.txt
  10. 1
      lists/france.txt
  11. 1
      lists/idaho.txt
  12. 1
      lists/mexico.txt
  13. 1
      lists/michigan.txt
  14. 6
      lists/midwest.txt
  15. 1
      lists/netherlands.txt
  16. 1
      lists/north_america.txt
  17. 1
      lists/ohio.txt
  18. 1
      lists/ontario.txt
  19. 13
      lists/osm/canada.txt
  20. 1
      lists/osm/mexico.txt
  21. 6
      lists/osm/midwest.txt
  22. 1
      lists/osm/ontario.txt
  23. 51
      lists/osm/usa.txt
  24. 1
      lists/pbf/australia_oceania.txt
  25. 13
      lists/pbf/canada.txt
  26. 2
      lists/pbf/central_america.txt
  27. 1
      lists/pbf/idaho.txt
  28. 1
      lists/pbf/mexico.txt
  29. 1
      lists/pbf/michigan.txt
  30. 6
      lists/pbf/midwest.txt
  31. 1
      lists/pbf/ontario.txt
  32. 1
      lists/pbf/pennsylvania.txt
  33. 1
      lists/pbf/south_america.txt
  34. 51
      lists/pbf/usa.txt
  35. 1
      lists/pennsylvania.txt
  36. 1
      lists/south_america.txt
  37. 1
      lists/spain.txt
  38. 4
      lists/test.txt
  39. 51
      lists/usa.txt
  40. 1
      lists/usa_midwest.txt
  41. 1
      lists/usa_northeast.txt
  42. 1
      lists/usa_pacific.txt
  43. 1
      lists/usa_south.txt
  44. 1
      lists/usa_west.txt
  45. 25
      manual_processing.md
  46. 13
      process_all.sh
  47. 128
      process_maps.py
  48. 0
      pymapsforge/__init__.py
  49. 63
      pymapsforge/download.py
  50. 32
      pymapsforge/md5_map.py
  51. 92
      pymapsforge/poly.py
  52. 141
      pymapsforge/process.py
  53. 17
      sea.osm
  54. 434
      shape2osm.py

8
.gitignore

@ -1,6 +1,12 @@
__pycache__
out/
out
test/
test
tmp/
*.swp
tmp
cache/
cache
logs/
logs
*.swp

82
README.md

@ -10,7 +10,8 @@ To run this program you'll need the following
* Python 3.x installed
* Java 1.8 or higher installed (OpenJDK works)
* wget installed
* bunzip2 installed (usually part of bzip2 package)
* gdal-bin
* python3-gdal (Ubuntu) installed
# Installation
@ -23,47 +24,52 @@ To run the program, cd to the directory where you extracted the release and run
### Program Usage
```
usage: process_maps.py [-h] [--map-list MAP_LIST] [--no-sleep] [--use-ram]
usage: process_maps.py [-h] --map-list MAP_LIST [--use-ram]
[--max-heap-space MAX_HEAP_SPACE]
[--output-map-name OUTPUT_MAP_NAME]
[--cached-maps-dir CACHED_MAPS_DIR]
[--output-map-name OUTPUT_MAP_NAME] --maps-dir MAPS_DIR
[--no-map-download]
optional arguments:
-h, --help show this help message and exit
--map-list MAP_LIST a text file with one map URL per line, can be
specified more than once
--no-sleep don't sleep between downloads -- WARNING you can
easily run into throttling on mirrors if you use this
option
--use-ram use RAM for mapsforge processing -- WARNING mapsforge
uses 10x the map size in RAM for processing (ie. 100Mb
map = 1Gb RAM usage), you want a LOT of RAM for this
option
--max-heap-space MAX_HEAP_SPACE
set the max heap space for the JVM, use standard -Xmx
values, default (4g) should be fine if not using
values, default (1g) should be fine if not using
--use-ram argument
--output-map-name OUTPUT_MAP_NAME
set the output .map and .poi file names
--cached-maps-dir CACHED_MAPS_DIR
Use cached maps in the specified directory instead of
downloads using map lists
--maps-dir MAPS_DIR Where downloaded maps will be stored/read from
--no-map-download Do NOT download maps, re-use maps from maps-dir
```
### Examples
```
./process_maps.py --max-heap-space 24g \
--output-map-name central_america \
--cached-maps-dir ~/osmmapdata/cache/central_america/20180225-1429
./process_maps.py --max-heap-space 24g \
--output-map-name midwest \
--cached-maps-dir ~/osmmapdata/cache/midwest/20180225-1429
#./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt
#./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt
./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name netherlands --map-list lists/netherlands.txt
./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name spain --map-list lists/spain.txt
./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name france --map-list lists/france.txt
./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia_oceania --map-list lists/australia_oceania.txt
./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia --map-list lists/australia.txt
./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name canada_ontario --map-list lists/ontario.txt
./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name local --map-list lists/midwest.txt --map-list lists/ontario.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_midwest --map-list lists/usa_midwest.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_northeast --map-list lists/usa_northeast.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_pacific --map-list lists/usa_pacific.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_south --map-list lists/usa_south.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_west --map-list lists/usa_west.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa --map-list lists/usa.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name canada --map-list lists/canada.txt
./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name north_america --map-list lists/north_america.txt
./process_maps.py --max-heap-space 24g \
--output-map-name south_america \
--cached-maps-dir ~/osmmapdata/cache/south_america/20180225-1429
```
# Caching maps for re-use / re-processing
@ -73,37 +79,27 @@ Included in the repo there is a ```download_maps.py``` script that is meant to h
### Program usage
```
usage: download_maps.py [-h] [--map-list MAP_LIST] [--no-sleep]
[--output-map-name OUTPUT_MAP_NAME]
[--cached-maps-dir CACHED_MAPS_DIR]
usage: download_maps.py [-h] --map-list MAP_LIST --map-dir MAP_DIR
optional arguments:
-h, --help show this help message and exit
--map-list MAP_LIST a text file with one map URL per line, can be
specified more than once
--no-sleep don't sleep between downloads -- WARNING you can
easily run into throttling on mirrors if you use this
option
--output-map-name OUTPUT_MAP_NAME
set the name of the map directory before Ymd-HM
--cached-maps-dir CACHED_MAPS_DIR
The root directory where maps should be cached
-h, --help show this help message and exit
--map-list MAP_LIST a text file with one map URL per line, can be specified
more than once
--map-dir MAP_DIR The directory where maps should be downloaded
```
### Example Usage
```
./download_maps.py --cached-maps-dir ./cache \
--output-map-name central_america \
--map-list lists/pbf/central_america.txt
./download_maps.py --map-dir ./cache \
--map-list lists/central_america.txt
./download_maps.py --cached-maps-dir ./cache \
--output-map-name midwest \
--map-list lists/pbf/midwest.txt
./download_maps.py --map-dir ./cache \
--map-list lists/midwest.txt
./download_maps.py --cached-maps-dir ./cache \
--output-map-name south_america \
--map-list lists/pbf/south_america.txt
./download_maps.py --map-dir ./cache \
--map-list lists/south_america.txt
```
# Licencing

15
azure_notes.md

@ -82,22 +82,19 @@ Download maps using lists to local cache (use a blob storage endpoint, these can
```
# Example, reformat to your needs/desires
./download_maps.py --map-list lists/test.txt --cached-maps-dir ./cache --output-map-name test
./download_maps.py --map-list lists/test.txt --map-dir ./cache
```
### Midwest USA
```
./process_maps.py --use-ram --max-heap-space 48g \
--output-map-name midwest \
--cached-maps-dir ~/osmmapdata/cache/midwest/20180225-1429
./process_maps.py --use-ram --max-heap-space 50g --maps-dir cache --no-map-download --output-map-name midwest --map-list lists/midwest.txt --map-list lists/ontario.txt
```
### South America
```
./process_maps.py --use-ram --max-heap-space 48g \
--output-map-name south_america \
--cached-maps-dir ~/osmmapdata/cache/south_america/20180225-1429
./process_maps.py --use-ram --max-heap-space 50g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt
```
### Central America
@ -105,8 +102,6 @@ Download maps using lists to local cache (use a blob storage endpoint, these can
*NOTE: This CAN NOT fit into 64Gb RAM, you'll need to setup a 128Gb SSD/Disk at ~/osm_map_processing/tmp for this to not enounter heap space errors*
```
./process_maps.py --max-heap-space 48g \
--output-map-name central_america \
--cached-maps-dir ~/osmmapdata/cache/central_america/20180225-1429
./process_maps.py --max-heap-space 50g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt
```

65
download_maps.py

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2016 Mike "KemoNine" Crosson
# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -12,67 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess, sys, os, pprint, datetime, argparse, time
base_path = os.path.dirname(os.path.realpath(__file__))
env = os.environ.copy()
FNULL = open(os.devnull, 'w')
wget_cmd = 'wget'
bunzip2_cmd = 'bunzip2'
import argparse
from pymapsforge import download
if __name__ == '__main__':
current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
parser = argparse.ArgumentParser()
parser.add_argument('--map-list', action='append',
parser.add_argument('--map-list', action='append', required=True,
help='a text file with one map URL per line, can be specified more than once')
parser.add_argument('--no-sleep', action='store_true',
help='don\'t sleep between downloads -- WARNING you can easily run into throttling on mirrors if you use this option')
parser.add_argument('--output-map-name', action='store', default='output',
help='set the name of the map directory before Ymd-HM')
parser.add_argument('--cached-maps-dir', action='store',
help='The root directory where maps should be cached')
parser.add_argument('--map-dir', action='store', required=True,
help='The directory where maps should be downloaded')
args = parser.parse_args()
if args.map_list is None or args.cached_maps_dir is None:
print('You MUST specify at least one map-list AND cached-maps-dir')
sys.exit(1)
# Normalize map path directory based on CLI arg ahead of any path manipulations
map_dl_dirs = [args.cached_maps_dir]
if args.output_map_name is not None:
map_dl_dirs.append(args.output_map_name)
map_dl_dirs.append(current_timestamp)
cached_maps_dir = os.path.abspath(os.path.join(*map_dl_dirs))
print('Downloading maps to : ' + cached_maps_dir)
map_list = []
if args.map_list is not None:
for alist in args.map_list:
with open(alist, 'r') as maps:
for line in maps:
map_list.append(line.strip())
if args.map_list is not None:
print('Downloading maps')
for line in map_list:
print(' ', end='')
print(line)
subprocess.run([wget_cmd, '-P', cached_maps_dir, line.strip()], stdout=FNULL, stderr=subprocess.STDOUT)
if not args.no_sleep:
print(' Sleeping to prevent throttle/blocking')
time.sleep(300) # Seconds
print('Decompressing maps (if necessary)')
for dirpath, dirnames, filenames in os.walk(cached_maps_dir):
for file in filenames:
if file.endswith('bz2'):
print(' ', end='')
print(file)
subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)])
download.download(args.map_dir, args.map_list)

1
lists/australia.txt

@ -0,0 +1 @@
australia-oceania/australia

1
lists/australia_oceania.txt

@ -0,0 +1 @@
australia-oceania

1
lists/canada.txt

@ -0,0 +1 @@
north-america/canada

2
lists/central_america.txt

@ -0,0 +1,2 @@
central-america
north-america/mexico

1
lists/deleware.txt

@ -0,0 +1 @@
north-america/us/delaware

1
lists/france.txt

@ -0,0 +1 @@
europe/france

1
lists/idaho.txt

@ -0,0 +1 @@
north-america/us/idaho

1
lists/mexico.txt

@ -0,0 +1 @@
north-america/mexico

1
lists/michigan.txt

@ -0,0 +1 @@
north-america/us/michigan

6
lists/midwest.txt

@ -0,0 +1,6 @@
north-america/us/illinois
north-america/us/indiana
north-america/us/michigan
north-america/us/minnesota
north-america/us/ohio
north-america/us/wisconsin

1
lists/netherlands.txt

@ -0,0 +1 @@
europe/netherlands

1
lists/north_america.txt

@ -0,0 +1 @@
north-america

1
lists/ohio.txt

@ -0,0 +1 @@
north-america/us/ohio

1
lists/ontario.txt

@ -0,0 +1 @@
north-america/canada/ontario

13
lists/osm/canada.txt

@ -1,13 +0,0 @@
http://download.geofabrik.de/north-america/canada/alberta-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/british-columbia-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/manitoba-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/new-brunswick-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/newfoundland-and-labrador-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/northwest-territories-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/nova-scotia-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/nunavut-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/quebec-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/saskatchewan-latest.osm.bz2
http://download.geofabrik.de/north-america/canada/yukon-latest.osm.bz2

1
lists/osm/mexico.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/mexico-latest.osm.bz2

6
lists/osm/midwest.txt

@ -1,6 +0,0 @@
http://download.geofabrik.de/north-america/us/illinois-latest.osm.bz2
http://download.geofabrik.de/north-america/us/indiana-latest.osm.bz2
http://download.geofabrik.de/north-america/us/michigan-latest.osm.bz2
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.bz2
http://download.geofabrik.de/north-america/us/ohio-latest.osm.bz2
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.bz2

1
lists/osm/ontario.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.bz2

51
lists/osm/usa.txt

@ -1,51 +0,0 @@
http://download.geofabrik.de/north-america/us/alabama-latest.osm.bz2
http://download.geofabrik.de/north-america/us/alaska-latest.osm.bz2
http://download.geofabrik.de/north-america/us/arizona-latest.osm.bz2
http://download.geofabrik.de/north-america/us/arkansas-latest.osm.bz2
http://download.geofabrik.de/north-america/us/california-latest.osm.bz2
http://download.geofabrik.de/north-america/us/colorado-latest.osm.bz2
http://download.geofabrik.de/north-america/us/connecticut-latest.osm.bz2
http://download.geofabrik.de/north-america/us/delaware-latest.osm.bz2
http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.bz2
http://download.geofabrik.de/north-america/us/florida-latest.osm.bz2
http://download.geofabrik.de/north-america/us/georgia-latest.osm.bz2
http://download.geofabrik.de/north-america/us/hawaii-latest.osm.bz2
http://download.geofabrik.de/north-america/us/idaho-latest.osm.bz2
http://download.geofabrik.de/north-america/us/illinois-latest.osm.bz2
http://download.geofabrik.de/north-america/us/indiana-latest.osm.bz2
http://download.geofabrik.de/north-america/us/iowa-latest.osm.bz2
http://download.geofabrik.de/north-america/us/kansas-latest.osm.bz2
http://download.geofabrik.de/north-america/us/kentucky-latest.osm.bz2
http://download.geofabrik.de/north-america/us/louisiana-latest.osm.bz2
http://download.geofabrik.de/north-america/us/maine-latest.osm.bz2
http://download.geofabrik.de/north-america/us/maryland-latest.osm.bz2
http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2
http://download.geofabrik.de/north-america/us/michigan-latest.osm.bz2
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.bz2
http://download.geofabrik.de/north-america/us/mississippi-latest.osm.bz2
http://download.geofabrik.de/north-america/us/missouri-latest.osm.bz2
http://download.geofabrik.de/north-america/us/montana-latest.osm.bz2
http://download.geofabrik.de/north-america/us/nebraska-latest.osm.bz2
http://download.geofabrik.de/north-america/us/nevada-latest.osm.bz2
http://download.geofabrik.de/north-america/us/new-hampshire-latest.osm.bz2
http://download.geofabrik.de/north-america/us/new-jersey-latest.osm.bz2
http://download.geofabrik.de/north-america/us/new-mexico-latest.osm.bz2
http://download.geofabrik.de/north-america/us/new-york-latest.osm.bz2
http://download.geofabrik.de/north-america/us/north-carolina-latest.osm.bz2
http://download.geofabrik.de/north-america/us/north-dakota-latest.osm.bz2
http://download.geofabrik.de/north-america/us/ohio-latest.osm.bz2
http://download.geofabrik.de/north-america/us/oklahoma-latest.osm.bz2
http://download.geofabrik.de/north-america/us/oregon-latest.osm.bz2
http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.bz2
http://download.geofabrik.de/north-america/us/rhode-island-latest.osm.bz2
http://download.geofabrik.de/north-america/us/south-carolina-latest.osm.bz2
http://download.geofabrik.de/north-america/us/south-dakota-latest.osm.bz2
http://download.geofabrik.de/north-america/us/tennessee-latest.osm.bz2
http://download.geofabrik.de/north-america/us/texas-latest.osm.bz2
http://download.geofabrik.de/north-america/us/utah-latest.osm.bz2
http://download.geofabrik.de/north-america/us/vermont-latest.osm.bz2
http://download.geofabrik.de/north-america/us/virginia-latest.osm.bz2
http://download.geofabrik.de/north-america/us/washington-latest.osm.bz2
http://download.geofabrik.de/north-america/us/west-virginia-latest.osm.bz2
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.bz2
http://download.geofabrik.de/north-america/us/wyoming-latest.osm.bz2

1
lists/pbf/australia_oceania.txt

@ -1 +0,0 @@
http://download.geofabrik.de/australia-oceania-latest.osm.pbf

13
lists/pbf/canada.txt

@ -1,13 +0,0 @@
http://download.geofabrik.de/north-america/canada/alberta-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/british-columbia-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/manitoba-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/new-brunswick-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/newfoundland-and-labrador-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/northwest-territories-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/nova-scotia-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/nunavut-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/quebec-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/saskatchewan-latest.osm.pbf
http://download.geofabrik.de/north-america/canada/yukon-latest.osm.pbf

2
lists/pbf/central_america.txt

@ -1,2 +0,0 @@
http://download.openstreetmap.fr/extracts/central-america-latest.osm.pbf
http://download.geofabrik.de/north-america/mexico-latest.osm.pbf

1
lists/pbf/idaho.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/us/idaho-latest.osm.pbf

1
lists/pbf/mexico.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/mexico-latest.osm.pbf

1
lists/pbf/michigan.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf

6
lists/pbf/midwest.txt

@ -1,6 +0,0 @@
http://download.geofabrik.de/north-america/us/illinois-latest.osm.pbf
http://download.geofabrik.de/north-america/us/indiana-latest.osm.pbf
http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.pbf
http://download.geofabrik.de/north-america/us/ohio-latest.osm.pbf
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.pbf

1
lists/pbf/ontario.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.pbf

1
lists/pbf/pennsylvania.txt

@ -1 +0,0 @@
http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.pbf

1
lists/pbf/south_america.txt

@ -1 +0,0 @@
http://download.openstreetmap.fr/extracts/south-america-latest.osm.pbf

51
lists/pbf/usa.txt

@ -1,51 +0,0 @@
http://download.geofabrik.de/north-america/us/alabama-latest.osm.pbf
http://download.geofabrik.de/north-america/us/alaska-latest.osm.pbf
http://download.geofabrik.de/north-america/us/arizona-latest.osm.pbf
http://download.geofabrik.de/north-america/us/arkansas-latest.osm.pbf
http://download.geofabrik.de/north-america/us/california-latest.osm.pbf
http://download.geofabrik.de/north-america/us/colorado-latest.osm.pbf
http://download.geofabrik.de/north-america/us/connecticut-latest.osm.pbf
http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf
http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.pbf
http://download.geofabrik.de/north-america/us/florida-latest.osm.pbf
http://download.geofabrik.de/north-america/us/georgia-latest.osm.pbf
http://download.geofabrik.de/north-america/us/hawaii-latest.osm.pbf
http://download.geofabrik.de/north-america/us/idaho-latest.osm.pbf
http://download.geofabrik.de/north-america/us/illinois-latest.osm.pbf
http://download.geofabrik.de/north-america/us/indiana-latest.osm.pbf
http://download.geofabrik.de/north-america/us/iowa-latest.osm.pbf
http://download.geofabrik.de/north-america/us/kansas-latest.osm.pbf
http://download.geofabrik.de/north-america/us/kentucky-latest.osm.pbf
http://download.geofabrik.de/north-america/us/louisiana-latest.osm.pbf
http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf
http://download.geofabrik.de/north-america/us/maryland-latest.osm.pbf
http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.pbf
http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.pbf
http://download.geofabrik.de/north-america/us/mississippi-latest.osm.pbf
http://download.geofabrik.de/north-america/us/missouri-latest.osm.pbf
http://download.geofabrik.de/north-america/us/montana-latest.osm.pbf
http://download.geofabrik.de/north-america/us/nebraska-latest.osm.pbf
http://download.geofabrik.de/north-america/us/nevada-latest.osm.pbf
http://download.geofabrik.de/north-america/us/new-hampshire-latest.osm.pbf
http://download.geofabrik.de/north-america/us/new-jersey-latest.osm.pbf
http://download.geofabrik.de/north-america/us/new-mexico-latest.osm.pbf
http://download.geofabrik.de/north-america/us/new-york-latest.osm.pbf
http://download.geofabrik.de/north-america/us/north-carolina-latest.osm.pbf
http://download.geofabrik.de/north-america/us/north-dakota-latest.osm.pbf
http://download.geofabrik.de/north-america/us/ohio-latest.osm.pbf
http://download.geofabrik.de/north-america/us/oklahoma-latest.osm.pbf
http://download.geofabrik.de/north-america/us/oregon-latest.osm.pbf
http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.pbf
http://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf
http://download.geofabrik.de/north-america/us/south-carolina-latest.osm.pbf
http://download.geofabrik.de/north-america/us/south-dakota-latest.osm.pbf
http://download.geofabrik.de/north-america/us/tennessee-latest.osm.pbf
http://download.geofabrik.de/north-america/us/texas-latest.osm.pbf
http://download.geofabrik.de/north-america/us/utah-latest.osm.pbf
http://download.geofabrik.de/north-america/us/vermont-latest.osm.pbf
http://download.geofabrik.de/north-america/us/virginia-latest.osm.pbf
http://download.geofabrik.de/north-america/us/washington-latest.osm.pbf
http://download.geofabrik.de/north-america/us/west-virginia-latest.osm.pbf
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.pbf
http://download.geofabrik.de/north-america/us/wyoming-latest.osm.pbf

1
lists/pennsylvania.txt

@ -0,0 +1 @@
north-america/us/pennsylvania

1
lists/south_america.txt

@ -0,0 +1 @@
south-america

1
lists/spain.txt

@ -0,0 +1 @@
europe/spain

4
lists/test.txt

@ -1,2 +1,2 @@
http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.pbf
http://download.geofabrik.de/north-america/us/delaware-latest.osm.bz2
north-america/canada/prince-edward-island
north-america/us/delaware

51
lists/usa.txt

@ -0,0 +1,51 @@
north-america/us/alabama
north-america/us/alaska
north-america/us/arizona
north-america/us/arkansas
north-america/us/california
north-america/us/colorado
north-america/us/connecticut
north-america/us/delaware
north-america/us/district-of-columbia
north-america/us/florida
north-america/us/georgia
north-america/us/hawaii
north-america/us/idaho
north-america/us/illinois
north-america/us/indiana
north-america/us/iowa
north-america/us/kansas
north-america/us/kentucky
north-america/us/louisiana
north-america/us/maine
north-america/us/maryland
north-america/us/massachusetts
north-america/us/michigan
north-america/us/minnesota
north-america/us/mississippi
north-america/us/missouri
north-america/us/montana
north-america/us/nebraska
north-america/us/nevada
north-america/us/new-hampshire
north-america/us/new-jersey
north-america/us/new-mexico
north-america/us/new-york
north-america/us/north-carolina
north-america/us/north-dakota
north-america/us/ohio
north-america/us/oklahoma
north-america/us/oregon
north-america/us/pennsylvania
north-america/us/rhode-island
north-america/us/south-carolina
north-america/us/south-dakota
north-america/us/tennessee
north-america/us/texas
north-america/us/utah
north-america/us/vermont
north-america/us/virginia
north-america/us/washington
north-america/us/west-virginia
north-america/us/wisconsin
north-america/us/wyoming

1
lists/usa_midwest.txt

@ -0,0 +1 @@
north-america/us-midwest

1
lists/usa_northeast.txt

@ -0,0 +1 @@
north-america/us-northeast

1
lists/usa_pacific.txt

@ -0,0 +1 @@
north-america/us-pacific

1
lists/usa_south.txt

@ -0,0 +1 @@
north-america/us-south

1
lists/usa_west.txt

@ -0,0 +1 @@
north-america/us-west

25
manual_processing.md

@ -1,25 +0,0 @@
Manual Procedure
=
Prep
==
- Download updated maps: http://download.geofabrik.de/north-america.html and http://download.geofabrik.de/north-america/canada.html
- Pull latest osmosis from http://wiki.openstreetmap.org/wiki/Osmosis
- Pull latest mapsforge for osmosis from https://github.com/mapsforge/mapsforge/blob/master/docs/Downloads.md
Process maps for device
==
- Setup osmosis in a directory (unzip release -- ZIPBOMB ALERT!!!!)
- Create plugins directory under osmosis
- Move mapsforge release jar to osmosis/lib/default/
- Decompress maps (if they were previously compressed)
- ./osmosis bin/osmosis --rb file=../maps/a_map.osm.pbf --mapfile-writer file=../maps/a_map.map
* If you run out of heap space add JAVACMD_OPTIONS="-Xmx16g" to the start of the line
* If increasing the heap doesn't work, append type=hd to end of command
* You can merge maps using --merge passed to osmosis (before --mapfile-writer) using n-1 instances of --merge where n is the number of --rb or --rx options passed
General Notes
=
Used to generate above info
* https://github.com/mapsforge/mapsforge/blob/master/docs/Getting-Started-Map-Writer.md

13
process_all.sh

@ -0,0 +1,13 @@
#!/bin/bash
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name local --map-list lists/michigan.txt --map-list lists/ontario.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name midwest --map-list lists/midwest.txt --map-list lists/ontario.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia_oceania --map-list lists/australia_oceania.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name usa --map-list lists/usa.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name canada --map-list lists/canada.txt
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name north_america --map-list lists/usa.txt --map-list lists/canada.txt

128
process_maps.py

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2016 Mike "KemoNine" Crosson
# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -12,124 +12,38 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess, sys, os, pprint, datetime, argparse, time
base_path = os.path.dirname(os.path.realpath(__file__))
env = os.environ.copy()
FNULL = open(os.devnull, 'w')
wget_cmd = 'wget'
bunzip2_cmd = 'bunzip2'
import argparse, datetime, os
from pymapsforge import process, download
if __name__ == '__main__':
current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
parser = argparse.ArgumentParser()
parser.add_argument('--map-list', action='append',
parser.add_argument('--map-list', action='append', required=True,
help='a text file with one map URL per line, can be specified more than once')
parser.add_argument('--no-sleep', action='store_true',
help='don\'t sleep between downloads -- WARNING you can easily run into throttling on mirrors if you use this option')
parser.add_argument('--use-ram', action='store_true',
help='use RAM for mapsforge processing -- WARNING mapsforge uses 10x the map size in RAM for processing (ie. 100Mb map = 1Gb RAM usage), you want a LOT of RAM for this option')
parser.add_argument('--max-heap-space', action='store', default='4g',
help='set the max heap space for the JVM, use standard -Xmx values, default (4g) should be fine if not using --use-ram argument')
parser.add_argument('--max-heap-space', action='store', default='1g',
help='set the max heap space for the JVM, use standard -Xmx values, default (1g) should be fine if not using --use-ram argument')
parser.add_argument('--output-map-name', action='store', default='output',
help='set the output .map and .poi file names')
parser.add_argument('--cached-maps-dir', action='store',
help='Use cached maps in the specified directory instead of downloads using map lists')
#TODO: Add argument to pass path to osmosis
#TODO: Add argument to pass path to output dir
parser.add_argument('--maps-dir', action='store', required=True,
help='Where downloaded maps will be stored/read from')
parser.add_argument('--no-map-download', action='store_true',
help='Do NOT download maps, re-use maps from maps-dir')
args = parser.parse_args()
if args.map_list is None and args.cached_maps_dir is None:
print('You MUST specify at least one map-list or cached-maps-dir')
sys.exit(1)
# Normalize map path directory based on CLI arg ahead of any path manipulations
# DEFAULTS TO 'dl' (non-normalized) to retain original behavior of script
cached_maps_dir = 'dl'
if args.cached_maps_dir is not None:
cached_maps_dir = os.path.abspath(args.cached_maps_dir)
map_list = []
if args.map_list is not None:
for alist in args.map_list:
with open(alist, 'r') as maps:
for line in maps:
map_list.append(line.strip())
print('Creating working directories')
if not os.path.exists('tmp'):
os.makedirs('tmp')
if not os.path.exists('out'):
os.makedirs('out')
os.chdir('out')
output_dir = current_timestamp + '-' + args.output_map_name
if not os.path.exists(output_dir):
os.makedirs(output_dir)
os.chdir(output_dir)
if args.map_list is not None:
if not os.path.exists('dl'):
os.makedirs('dl')
print('Downloading maps')
for line in map_list:
print(' ', end='')
print(line)
subprocess.run([wget_cmd, '-P', 'dl', line.strip()], stdout=FNULL, stderr=subprocess.STDOUT)
if not args.no_sleep:
print(' Sleeping to prevent throttle/blocking')
time.sleep(300) # Seconds
print('Decompressing maps (if necessary)')
for dirpath, dirnames, filenames in os.walk('dl'):
for file in filenames:
if file.endswith('bz2'):
print(' ', end='')
print(file)
subprocess.run([bunzip2_cmd, os.path.join(dirpath, file)])
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
env['JAVACMD_OPTIONS'] = '-Xmx' + args.max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
print('Finding maps to process')
files_to_process = []
for dirpath, dirnames, filenames in os.walk(cached_maps_dir):
for file in filenames:
print(' Found map: ', end='')
print(os.path.join(dirpath, file))
files_to_process.append(os.path.join(dirpath, file))
current_timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M')
base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__))))
args.maps_dir = os.path.abspath(args.maps_dir)
print('Processing maps using osmosis')
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
for file in files_to_process:
if file.endswith('osm'):
osmosis_cmd.extend(['--rx', 'file=' + file])
elif file.endswith('pbf'):
osmosis_cmd.extend(['--rb', 'file=' + file])
for x in range(0, len(files_to_process) - 1):
osmosis_cmd.append('--merge')
osmosis_cmd.extend(['--mapfile-writer', 'file=' + args.output_map_name + '.map'])
if args.use_ram:
osmosis_cmd.extend(['type=ram'])
else:
osmosis_cmd.extend(['type=hd'])
osmosis_cmd.extend(['threads=4'])
cmd = subprocess.Popen(osmosis_cmd, env=env)
cmd.wait()
process.setup_working_dirs(base_path, current_timestamp, args.output_map_name)
if not args.no_map_download:
download.download(args.maps_dir, args.map_list)
print('Processing POIs using osmosis')
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
for file in files_to_process:
if file.endswith('osm'):
osmosis_cmd.extend(['--rx', 'file=' + file])
elif file.endswith('pbf'):
osmosis_cmd.extend(['--rb', 'file=' + file])
for x in range(0, len(files_to_process) - 1):
osmosis_cmd.append('--merge')
osmosis_cmd.extend(['--poi-writer', 'file=' + args.output_map_name + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')])
cmd = subprocess.Popen(osmosis_cmd, env=env)
cmd.wait()
process.prep_land_sea_polys(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name)
process.process_land_sea_polys(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name, args.max_heap_space, args.use_ram)
process.osmosis_main(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name, args.max_heap_space, args.use_ram)
process.osmosis_pois(base_path, current_timestamp, args.maps_dir, args.map_list, args.output_map_name, args.max_heap_space)

0
pymapsforge/__init__.py

63
pymapsforge/download.py

@ -0,0 +1,63 @@
# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess, sys, os, pprint, datetime, argparse, time
from . import md5_map
MIRROR_BASE_URL = 'http://download.geofabrik.de/'
MIRROR_FILE_SUFFIX = '-latest.osm.pbf'
MIRROR_MD5_SUFFIX = MIRROR_FILE_SUFFIX + '.md5'
MIRROR_POLY_SUFFIX = '.poly'
def download(dl_dir, dl_lists):
FNULL = open(os.devnull, 'w')
wget_cmd = 'wget'
bunzip2_cmd = 'bunzip2'
cached_maps_dir = os.path.abspath(dl_dir)
print('Downloading maps to : ' + cached_maps_dir)
map_list = []
for alist in dl_lists:
with open(alist, 'r') as maps:
for line in maps:
map_list.append(line.strip())
print('Downloading land polygons from data.openstreetmapdata.com')
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, 'http://data.openstreetmapdata.com/land-polygons-split-4326.zip'], stdout=FNULL, stderr=subprocess.STDOUT)
print('Downloading maps (.osm.pbf, .md5, .poly)')
first_iteration = True
for amap in map_list:
if not first_iteration:
print(' Sleeping to prevent throttle/blocking')
time.sleep(30) # Seconds
else:
first_iteration = False
print(' ', end='')
print(amap)
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_FILE_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_MD5_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_POLY_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
print('Checking map md5s')
for amap in map_list:
map_root = amap.strip().split('/')[-1]
map_file = os.path.join(cached_maps_dir, map_root + MIRROR_FILE_SUFFIX)
md5_file = os.path.join(cached_maps_dir, map_root + MIRROR_MD5_SUFFIX)
print(' ', end='')
print(amap + ': ', end='')
print(md5_map.check_hash(map_file, md5_file))

32
pymapsforge/md5_map.py

@ -0,0 +1,32 @@
# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Borrowed from the web: https://www.joelverhagen.com/blog/2011/02/md5-hash-of-file-in-python/
import hashlib
def md5Checksum(filePath):
with open(filePath, 'rb') as fh:
m = hashlib.md5()
while True:
data = fh.read(8192)
if not data:
break
m.update(data)
return m.hexdigest()
def check_hash(map_file, md5_file):
published_md5 = None
with open(md5_file, 'r') as f:
published_md5 = f.readline().strip().split()[0]
map_md5 = md5Checksum(map_file)
return published_md5 == map_md5

92
pymapsforge/poly.py

@ -0,0 +1,92 @@
# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Adapted from https://github.com/mapsforge/mapsforge-creator/blob/master/poly2bb.pl
import re, os
BOUND_BUFFER = 0.1
COORD_MATCHER = re.compile('^\s*([0-9.E+-]+)\s+([0-9.E+-]+)\s*$')
def calculate_total_bounds(maps_dir, map_lists):
cached_maps_dir = os.path.abspath(maps_dir)
x_min = 360.0
x_max = -360.0
y_min = 360.0
y_max = -360.0
for alist in map_lists:
with open(alist, 'r') as maps:
for line in maps:
current_map = line.strip().split('/')[-1]
bounds = calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly'))
if bounds['bottom'] < y_min:
y_min = bounds['bottom']
if bounds['left'] < x_min:
x_min = bounds['left']
if bounds['top'] > y_max:
y_max = bounds['top']
if bounds['right'] > x_max:
x_max = bounds['right']
values = {}
values['bottom'] = y_min
values['left'] = x_min
values['top'] = y_max
values['right'] = x_max
return values
def calculate_bounds(mappoly):
# Opposite of max possible values so we find the right bounding box later
x_min = 360.0
x_max = -360.0
y_min = 360.0
y_max = -360.0
with open(mappoly, 'r') as polyfile:
for line in polyfile:
if COORD_MATCHER.match(line.strip()):
values = line.split()
x = float(values[0])
y = float(values[1])
if x < x_min:
x_min = x
if x > x_max:
x_max = x
if y < y_min:
y_min = y
if y > y_max:
y_max = y
# Buffer bounds
x_min = x_min - BOUND_BUFFER
x_max = x_max + BOUND_BUFFER
y_min = y_min - BOUND_BUFFER
y_max = y_max + BOUND_BUFFER
# Cleanup bounding (per original script)
x_min = -180 if x_min < -180 else x_min
x_max = 180 if x_max > 180 else x_max
y_min = -90 if y_min < -90 else y_min
y_max = 90 if y_max > 90 else y_max
# Dict w/ useful values
values = {}
values['bottom'] = y_min
values['left'] = x_min
values['top'] = y_max
values['right'] = x_max
x_center = (x_min + x_max) / 2.0
y_center = (y_min + y_max) / 2.0
values['lat'] = y_center
values['lon'] = x_center
return values

141
pymapsforge/process.py

@ -0,0 +1,141 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2018 Mike "KemoNine" Crosson
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess, os, zipfile, shutil, fileinput, glob
from . import poly
FNULL = open(os.devnull, 'w')
SOURCE_MAP_POSTFIX = '-latest.osm.pbf'
def setup_working_dirs(base_path, current_timestamp, output_map_name):
print('Creating working directories as needed')
if not os.path.exists(os.path.join(base_path, 'tmp')):
os.makedirs(os.path.join(base_path, 'tmp'))
if not os.path.exists(os.path.join(base_path, 'out')):
os.makedirs(os.path.join(base_path, 'out'))
output_dir = current_timestamp + '-' + output_map_name
if not os.path.exists(os.path.join(base_path, 'out', output_dir)):
os.makedirs(os.path.join(base_path, 'out', output_dir))
def prep_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name):
print('Prepping land/sea polygons (one prep per map in list(s))')
output_dir = current_timestamp + '-' + output_map_name
polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
polys_zip = zipfile.ZipFile(os.path.join(maps_dir, 'land-polygons-split-4326.zip'))
for alist in map_lists:
with open(alist, 'r') as maps:
for line in maps:
map_polys_dir = os.path.join(polys_output_dir, line.strip().split('/')[-1])
print(' Extracting to: ' + map_polys_dir)
os.makedirs(map_polys_dir)
polys_zip.extractall(map_polys_dir)
polys_zip.close()
def process_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram):
print('Processing land/sea polygons (one process per map in list(s))')
output_dir = current_timestamp + '-' + output_map_name
polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys')
cached_maps_dir = os.path.abspath(maps_dir)
for alist in map_lists:
with open(alist, 'r') as maps:
for line in maps:
current_map = line.strip().split('/')[-1]
print(' Processing: ' + current_map)
bounds = poly.calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly'))
subprocess.run(['ogr2ogr', '-overwrite', '-progress', '-skipfailures', '-clipsrc', str(bounds['left']), str(bounds['bottom']), str(bounds['right']), str(bounds['top']), os.path.join(polys_output_dir, current_map + '.shp'), os.path.join(polys_output_dir, current_map, 'land-polygons-split-4326', 'land_polygons.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
subprocess.run(['python2', os.path.join(base_path, 'shape2osm.py'), '-l', os.path.join(polys_output_dir, current_map), os.path.join(polys_output_dir, current_map + '.shp')], stdout=FNULL, stderr=subprocess.STDOUT)
shutil.copyfile(os.path.join(base_path, 'sea.osm'), os.path.join(polys_output_dir, current_map + '_sea.osm'))
with fileinput.FileInput(os.path.join(polys_output_dir, current_map + '_sea.osm'), inplace=True, backup='.bak') as sea:
for line in sea:
updated_line = line.replace('$BOTTOM', str(bounds['bottom']))
updated_line = updated_line.replace('$LEFT', str(bounds['left']))
updated_line = updated_line.replace('$TOP', str(bounds['top']))
updated_line = updated_line.replace('$RIGHT', str(bounds['right']))
print(updated_line, end='')
map_list = []
for alist in map_lists:
with open(alist, 'r') as maps:
for line in maps:
map_list.append(line.strip().split('/')[-1])
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
env = os.environ.copy()
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
print('Processing land/sea polys using osmosis')
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
for amap in map_list:
osmosis_cmd.extend(['--rb', 'file=' + os.path.join(maps_dir, amap + SOURCE_MAP_POSTFIX)])
for apolymap in glob.glob(os.path.join(polys_output_dir, amap + '*.osm')):
print(' Found poly map : ' + apolymap)
osmosis_cmd.extend(['--rx', 'file=' + apolymap, '--s', '--m'])
osmosis_cmd.extend(['--log-progress'])
for x in range(0, len(map_list) - 1):
osmosis_cmd.append('--merge')
osmosis_cmd.extend(['--log-progress'])
osmosis_cmd.extend(['--wb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
osmosis_cmd.extend(['omitmetadata=true'])
cmd = subprocess.Popen(osmosis_cmd, env=env)
cmd.wait()
def osmosis_main(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram):
output_dir = current_timestamp + '-' + output_map_name
bounds = poly.calculate_total_bounds(maps_dir, map_lists)
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
env = os.environ.copy()
env['JAVACMD_OPTIONS'] = '-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
print('-----------------------------')
print('Processing maps using osmosis')
print('-----------------------------')
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
osmosis_cmd.extend(['--rb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf'])
osmosis_cmd.extend(['--log-progress'])
osmosis_cmd.extend(['--mapfile-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.map'])
if use_ram:
osmosis_cmd.extend(['type=ram'])
else:
osmosis_cmd.extend(['type=hd'])
osmosis_cmd.extend(['bbox=' + str(bounds['bottom']) + ',' + str(bounds['left']) + ',' + str(bounds['top']) + ',' + str(bounds['right'])])
osmosis_cmd.extend(['threads=1', 'map-start-zoom=8'])
cmd = subprocess.Popen(osmosis_cmd, env=env)
cmd.wait()
def osmosis_pois(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space):
output_dir = current_timestamp + '-' + output_map_name
map_list = []
for alist in map_lists:
with open(alist, 'r') as maps:
for line in maps:
map_list.append(os.path.join(maps_dir, line.strip().split('/')[-1]+SOURCE_MAP_POSTFIX))
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs)
env = os.environ.copy()
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss)
print('Processing POIs using osmosis')
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')]
for amap in map_list:
osmosis_cmd.extend(['--rb', 'file=' + amap])
for x in range(0, len(map_list) - 1):
osmosis_cmd.append('--merge')
osmosis_cmd.extend(['--poi-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')])
cmd = subprocess.Popen(osmosis_cmd, env=env)
cmd.wait()

17
sea.osm

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<osm version="0.6">
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459320" version="1" lon="$LEFT" lat="$BOTTOM" />
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459321" version="1" lon="$LEFT" lat="$TOP" />
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459322" version="1" lon="$RIGHT" lat="$TOP" />
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459323" version="1" lon="$RIGHT" lat="$BOTTOM" />
<way timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951623372" version="1">
<nd ref="32951459320" />
<nd ref="32951459321" />
<nd ref="32951459322" />
<nd ref="32951459323" />
<nd ref="32951459320" />
<tag k="area" v="yes" />
<tag k="layer" v="-5" />
<tag k="natural" v="sea" />
</way>
</osm>

434
shape2osm.py

@ -0,0 +1,434 @@
#!/usr/bin/python
"""
This script is designed to act as assistance in converting shapefiles
to OpenStreetMap data. This file is optimized and tested with MassGIS
shapefiles, converted to EPSG:4326 before being passed to the script.
You can perform this conversion with
ogr2ogr -t_srs EPSG:4326 new_file.shp old_file.shp
It is expected that you will modify the fixed_tags, tag_mapping, and
boring_tags attributes of this script before running. You should read,
or at least skim, the code up until it says:
DO NOT CHANGE AFTER THIS LINE.
to accomodate your own data.
Modified to support a maximum waylength and multiple outer ways in a
multipolygon relationship
adapted by devemux86
"""
__author__ = "Christopher Schmidt <crschmidt@crschmidt.net>"
__version__ = "$Id: polyshp2osm-WaterBody.py,v 1.3 2009/03/30 20:54:13 tbook Exp $"
gdal_install = """
Installing GDAL depends on your platform. Information is available at:
http://trac.osgeo.org/gdal/wiki/DownloadingGdalBinaries
For Debian-based systems:
apt-get install python-gdal
will usually suffice.
"""
# These tags are attached to all exterior ways. You can put any key/value pairs
# in this dictionary.
fixed_tags = {
'natural': 'nosea',
'layer': '-5'
}
# Here are a number of functions: These functions define tag mappings. The API
# For these functions is that they are passed the attributes from a feature,
# and they return a list of two-tuples which match to key/value pairs.
# The following Ftypes are not imported
# 46100 - Submerged Stream
ignoreField = "tile_x"
ignoreValues = []
# The following Ftypes are imported, but don't have any OSM tags attached:
# 36100 - Playa - An area from which water evaporates
def ftype(data):
"""Type of body - From NHD Ftype"""
natural = {
'LakePond': 'water',
'SwampMarsh': 'wetland'
}
if 'ftype' in data:
if data['ftype'] in natural:
return [('natural', natural[data['ftype']])]
return None
def fcode(data):
"""For features where the ftype is not specific enough"""
landuse = {
'43600': 'reservoir', #Reservoir
'43601': 'reservoir', #Reservoir
'43602': 'reservoir', #Reservoir
'43603': 'reservoir', #Reservoir
'43604': 'reservoir', #Reservoir
'43605': 'reservoir', #Reservoir
'43606': 'reservoir', #Reservoir
'43607': 'reservoir', #Reservoir
'43609': 'reservoir', #Reservoir
'43610': 'reservoir', #Reservoir
'43611': 'reservoir', #Reservoir
'43612': 'reservoir', #Reservoir
'43614': 'reservoir', #Reservoir
'43615': 'reservoir', #Reservoir
'43616': 'reservoir', #Reservoir
'43617': 'reservoir', #Reservoir
'43618': 'reservoir', #Reservoir
'43619': 'reservoir', #Reservoir
'43620': 'reservoir' #Reservoir
}
manmade = {
'43613': 'reservoir_covered' #Covered Reservoir
}
leisure = {
'43608': 'swimming_pool' #Swimming Pool
}
if 'fcode' in data:
keys = []
if data['fcode'] in landuse:
keys.append( ('landuse', landuse[data['fcode']]) )
if data['fcode'] in manmade:
keys.append( ('man_made', manmade[data['fcode']]) )
if data['fcode'] in leisure:
keys.append( ('leisure', leisure[data['fcode']]) )
return keys
return None
# The most important part of the code: define a set of key/value pairs
# to iterate over to generate keys. This is a list of two-tuples: first
# is a 'key', which is only used if the second value is a string. In
# that case, it is a map of lowercased fieldnames to OSM tag names: so
# fee_owner maps to 'owner' in the OSM output.
# if the latter is callable (has a __call__; is a function), then that
# method is called, passing in a dict of feature attributes with
# lowercased key names. Those functions can then return a list of
# two-tuples to be used as tags, or nothin' to skip the tags.
tag_mapping = [
('ftype', ftype),
('fcode', fcode),
('gnis_name', 'name'),
('gnis_id', 'gnis:feature_id'),
('elevation', 'ele'),
]
# These tags are not exported, even with the source data; this should be
# used for tags which are usually calculated in a GIS. AREA and LEN are
# common.
boring_tags = [ 'error', 'tile_x', 'tile_y' ]
# Namespace is used to prefix existing data attributes. If 'None', or
# '--no-source' is set, then source attributes are not exported, only
# attributes in tag_mapping.
namespace = "NHD"
#namespace = None
# Uncomment the "DONT_RUN = False" line to get started.
#DONT_RUN = True
DONT_RUN = False
# Set the maximum length of a way (in nodes) before it is split into
# shorter ways
Max_Waylength = 1500000
# =========== DO NOT CHANGE AFTER THIS LINE. ===========================
# Below here is regular code, part of the file. This is not designed to
# be modified by users.
# ======================================================================
import sys
try:
try:
from osgeo import ogr
except ImportError:
import ogr
except ImportError:
__doc__ += gdal_install
if DONT_RUN:
print __doc__
sys.exit(2)
print "OGR Python Bindings not installed.\n%s" % gdal_install
sys.exit(1)
def close_file():
""" Internal. Close an open file."""
global open_file
if not open_file.closed:
open_file.write("</osm>")
open_file.close()
def start_new_file():
""" Internal. Open a new file, closing existing file if neccesary."""
global open_file, file_counter
file_counter += 1
if open_file:
close_file()
open_file = open("%s%s.osm" % (file_name, file_counter), "w")
print >>open_file, "<osm version='0.6'>"
def clean_attr(val):
"""Internal. Hacky way to make attribute XML safe."""
val = str(val)
val = val.replace("&", "&amp;").replace("'", "&quot;").replace("<", "&lt;").replace(">", "&gt;").strip()
return val
def add_ring_nodes(ring):
"""Internal. Write the outer ring nodes."""
global open_file, id_counter
ringways = []
ids = []
firstnode = id_counter
if range(ring.GetPointCount() - 1) == 0 or ring.GetPointCount() == 0:
print >>sys.stderr, "Degenerate ring."
return
for count in range(ring.GetPointCount() - 1):
ids.append(id_counter)
print >>open_file, "<node timestamp='1969-12-31T23:59:59Z' changeset='-1' id='%s' version='1' lon='%s' lat='%s' />" % (id_counter, ring.GetX(count), ring.GetY(count))
id_counter += 1
if (count > 0) and ((count % (Max_Waylength - 1)) == 0):
ringways.append(ids)
ids = []
ids.append(id_counter - 1)
ids.append(firstnode)
ringways.append(ids)
return ringways
def add_ring_way(ring):
"""Internal. write out the 'holes' in a polygon."""
global open_file, id_counter
ids = []
ringways = []
for count in range(ring.GetPointCount() - 1):
ids.append(id_counter)
print >>open_file, "<node timestamp='1969-12-31T23:59:59Z' changeset='-1' version='1' id='%s' lon='%s' lat='%s' />" % (id_counter, ring.GetX(count), ring.GetY(count))
id_counter += 1
if len(ids) == 0:
return None
print >>open_file, "<way timestamp='1969-12-31T23:59:59Z' changeset='-1' id='%s' version='1'>" % id_counter
way_id = id_counter
id_counter += 1
count = 0
for i in ids:
print >>open_file, "<nd ref='%s' />" % i
count += 1
if count >= Max_Waylength - 1:
count = 0
print >>open_file, "</way>"
ringways.append(way_id)
print >>open_file, "<way timestamp='1969-12-31T23:59:59Z' changeset='-1'id='%s' version='1'>" % id_counter
way_id = id_counter
id_counter += 1
print >>open_file, "<nd ref='%s' />" % i
print >>open_file, "<nd ref='%s' />" % ids[0]
print >>open_file, "</way>"
ringways.append(way_id)
return ringways
open_file = None
file_name = None
id_counter = 22951459320
file_counter = 0
counter = 0
class AppError(Exception): pass
def run(filename, slice_count=1, obj_count=100000000, output_location=None, no_source=False):
"""Run the converter. Requires open_file, file_name, id_counter,
file_counter, counter to be defined in global space; not really a very good
singleton."""
global id_counter, file_counter, counter, file_name, open_file, namespace
if no_source:
namespace=None
if output_location:
file_name = output_location
ds = ogr.Open(filename)
if not ds:
raise AppError("OGR Could not open the file %s" % filename)
l = ds.GetLayer(0)
max_objs_per_file = obj_count
extent = l.GetExtent()
if extent[0] < -180 or extent[0] > 180 or extent[2] < -90 or extent[2] > 90:
raise AppError("Extent does not look like degrees; are you sure it is? \n(%s, %s, %s, %s)" % (extent[0], extent[2], extent[1], extent[3]))
slice_width = (extent[1] - extent[0]) / slice_count
seen = {}
print "Running %s slices with %s base filename against shapefile %s" % (
slice_count, file_name, filename)
for i in range(slice_count):
l.ResetReading()
l.SetSpatialFilterRect(extent[0] + slice_width * i, extent[2], extent[0] + (slice_width * (i + 1)), extent[3])
start_new_file()
f = l.GetNextFeature()
obj_counter = 0
last_obj_split = 0
while f:
start_id_counter = id_counter
if f.GetFID() in seen:
f = l.GetNextFeature()
continue
seen[f.GetFID()] = True
# done = False
# while f.GetField(ignoreField) in ignoreValues:
# if l.GetNextFeature() == None:
# done = True
# break
# f = l.GetNextFeature()
# if done == True:
# break
outerways = []
innerways = []
geom = f.GetGeometryRef()
ring = geom.GetGeometryRef(0)
objcount = ring.GetPointCount()
for i in range(1, geom.GetGeometryCount()):
objcount += geom.GetGeometryRef(i).GetPointCount()
objcount += 1
if (obj_counter - last_obj_split + objcount) > max_objs_per_file:
print "Splitting file with %s objs" % (obj_counter - last_obj_split)
start_new_file()
last_obj_split = obj_counter
if objcount > max_objs_per_file:
print "Warning: a feature contains %i objects which is more than the %i object limit. It will be placed in a file by itself." % (objcount, max_objs_per_file)