
54 changed files with 952 additions and 397 deletions
@ -1,6 +1,12 @@
|
||||
__pycache__ |
||||
out/ |
||||
out |
||||
test/ |
||||
test |
||||
tmp/ |
||||
*.swp |
||||
tmp |
||||
cache/ |
||||
cache |
||||
logs/ |
||||
logs |
||||
*.swp |
||||
|
@ -0,0 +1,6 @@
|
||||
north-america/us/illinois |
||||
north-america/us/indiana |
||||
north-america/us/michigan |
||||
north-america/us/minnesota |
||||
north-america/us/ohio |
||||
north-america/us/wisconsin |
@ -1,13 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/canada/alberta-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/british-columbia-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/manitoba-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/new-brunswick-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/newfoundland-and-labrador-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/northwest-territories-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/nova-scotia-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/nunavut-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/quebec-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/saskatchewan-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/canada/yukon-latest.osm.bz2 |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/mexico-latest.osm.bz2 |
@ -1,6 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/illinois-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/indiana-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/michigan-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/ohio-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.bz2 |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.bz2 |
@ -1,51 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/alabama-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/alaska-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/arizona-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/arkansas-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/california-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/colorado-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/connecticut-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/delaware-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/florida-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/georgia-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/hawaii-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/idaho-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/illinois-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/indiana-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/iowa-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/kansas-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/kentucky-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/louisiana-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/maine-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/maryland-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/michigan-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/mississippi-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/missouri-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/montana-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/nebraska-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/nevada-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/new-hampshire-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/new-jersey-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/new-mexico-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/new-york-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/north-carolina-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/north-dakota-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/ohio-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/oklahoma-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/oregon-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/rhode-island-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/south-carolina-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/south-dakota-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/tennessee-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/texas-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/utah-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/vermont-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/virginia-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/washington-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/west-virginia-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.bz2 |
||||
http://download.geofabrik.de/north-america/us/wyoming-latest.osm.bz2 |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/australia-oceania-latest.osm.pbf |
@ -1,13 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/canada/alberta-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/british-columbia-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/manitoba-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/new-brunswick-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/newfoundland-and-labrador-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/northwest-territories-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/nova-scotia-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/nunavut-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/quebec-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/saskatchewan-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/canada/yukon-latest.osm.pbf |
@ -1,2 +0,0 @@
|
||||
http://download.openstreetmap.fr/extracts/central-america-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/mexico-latest.osm.pbf |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/idaho-latest.osm.pbf |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/mexico-latest.osm.pbf |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf |
@ -1,6 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/illinois-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/indiana-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/ohio-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.pbf |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/canada/ontario-latest.osm.pbf |
@ -1 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.pbf |
@ -1 +0,0 @@
|
||||
http://download.openstreetmap.fr/extracts/south-america-latest.osm.pbf |
@ -1,51 +0,0 @@
|
||||
http://download.geofabrik.de/north-america/us/alabama-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/alaska-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/arizona-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/arkansas-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/california-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/colorado-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/connecticut-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/florida-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/georgia-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/hawaii-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/idaho-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/illinois-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/indiana-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/iowa-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/kansas-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/kentucky-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/louisiana-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/maryland-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/michigan-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/minnesota-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/mississippi-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/missouri-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/montana-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/nebraska-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/nevada-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/new-hampshire-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/new-jersey-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/new-mexico-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/new-york-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/north-carolina-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/north-dakota-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/ohio-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/oklahoma-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/oregon-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/pennsylvania-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/south-carolina-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/south-dakota-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/tennessee-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/texas-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/utah-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/vermont-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/virginia-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/washington-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/west-virginia-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/wisconsin-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/wyoming-latest.osm.pbf |
@ -1,2 +1,2 @@
|
||||
http://download.geofabrik.de/north-america/canada/prince-edward-island-latest.osm.pbf |
||||
http://download.geofabrik.de/north-america/us/delaware-latest.osm.bz2 |
||||
north-america/canada/prince-edward-island |
||||
north-america/us/delaware |
||||
|
@ -0,0 +1,51 @@
|
||||
north-america/us/alabama |
||||
north-america/us/alaska |
||||
north-america/us/arizona |
||||
north-america/us/arkansas |
||||
north-america/us/california |
||||
north-america/us/colorado |
||||
north-america/us/connecticut |
||||
north-america/us/delaware |
||||
north-america/us/district-of-columbia |
||||
north-america/us/florida |
||||
north-america/us/georgia |
||||
north-america/us/hawaii |
||||
north-america/us/idaho |
||||
north-america/us/illinois |
||||
north-america/us/indiana |
||||
north-america/us/iowa |
||||
north-america/us/kansas |
||||
north-america/us/kentucky |
||||
north-america/us/louisiana |
||||
north-america/us/maine |
||||
north-america/us/maryland |
||||
north-america/us/massachusetts |
||||
north-america/us/michigan |
||||
north-america/us/minnesota |
||||
north-america/us/mississippi |
||||
north-america/us/missouri |
||||
north-america/us/montana |
||||
north-america/us/nebraska |
||||
north-america/us/nevada |
||||
north-america/us/new-hampshire |
||||
north-america/us/new-jersey |
||||
north-america/us/new-mexico |
||||
north-america/us/new-york |
||||
north-america/us/north-carolina |
||||
north-america/us/north-dakota |
||||
north-america/us/ohio |
||||
north-america/us/oklahoma |
||||
north-america/us/oregon |
||||
north-america/us/pennsylvania |
||||
north-america/us/rhode-island |
||||
north-america/us/south-carolina |
||||
north-america/us/south-dakota |
||||
north-america/us/tennessee |
||||
north-america/us/texas |
||||
north-america/us/utah |
||||
north-america/us/vermont |
||||
north-america/us/virginia |
||||
north-america/us/washington |
||||
north-america/us/west-virginia |
||||
north-america/us/wisconsin |
||||
north-america/us/wyoming |
@ -1,25 +0,0 @@
|
||||
Manual Procedure |
||||
= |
||||
|
||||
Prep |
||||
== |
||||
- Download updated maps: http://download.geofabrik.de/north-america.html and http://download.geofabrik.de/north-america/canada.html |
||||
- Pull latest osmosis from http://wiki.openstreetmap.org/wiki/Osmosis |
||||
- Pull latest mapsforge for osmosis from https://github.com/mapsforge/mapsforge/blob/master/docs/Downloads.md |
||||
|
||||
Process maps for device |
||||
== |
||||
- Setup osmosis in a directory (unzip release -- ZIPBOMB ALERT!!!!) |
||||
- Create plugins directory under osmosis |
||||
- Move mapsforge release jar to osmosis/lib/default/ |
||||
- Decompress maps (if they were previously compressed) |
||||
- ./osmosis bin/osmosis --rb file=../maps/a_map.osm.pbf --mapfile-writer file=../maps/a_map.map |
||||
* If you run out of heap space add JAVACMD_OPTIONS="-Xmx16g" to the start of the line |
||||
* If increasing the heap doesn't work, append type=hd to end of command |
||||
* You can merge maps using --merge passed to osmosis (before --mapfile-writer) using n-1 instances of --merge where n is the number of --rb or --rx options passed |
||||
|
||||
General Notes |
||||
= |
||||
Used to generate above info |
||||
* https://github.com/mapsforge/mapsforge/blob/master/docs/Getting-Started-Map-Writer.md |
||||
|
@ -0,0 +1,13 @@
|
||||
#!/bin/bash |
||||
|
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name local --map-list lists/michigan.txt --map-list lists/ontario.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name midwest --map-list lists/midwest.txt --map-list lists/ontario.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia_oceania --map-list lists/australia_oceania.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name usa --map-list lists/usa.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name canada --map-list lists/canada.txt |
||||
./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name north_america --map-list lists/usa.txt --map-list lists/canada.txt |
||||
|
@ -0,0 +1,63 @@
|
||||
# Copyright 2018 Mike "KemoNine" Crosson |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import subprocess, sys, os, pprint, datetime, argparse, time |
||||
from . import md5_map |
||||
|
||||
MIRROR_BASE_URL = 'http://download.geofabrik.de/' |
||||
MIRROR_FILE_SUFFIX = '-latest.osm.pbf' |
||||
MIRROR_MD5_SUFFIX = MIRROR_FILE_SUFFIX + '.md5' |
||||
MIRROR_POLY_SUFFIX = '.poly' |
||||
|
||||
def download(dl_dir, dl_lists): |
||||
|
||||
FNULL = open(os.devnull, 'w') |
||||
|
||||
wget_cmd = 'wget' |
||||
bunzip2_cmd = 'bunzip2' |
||||
|
||||
cached_maps_dir = os.path.abspath(dl_dir) |
||||
|
||||
print('Downloading maps to : ' + cached_maps_dir) |
||||
|
||||
map_list = [] |
||||
|
||||
for alist in dl_lists: |
||||
with open(alist, 'r') as maps: |
||||
for line in maps: |
||||
map_list.append(line.strip()) |
||||
|
||||
print('Downloading land polygons from data.openstreetmapdata.com') |
||||
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, 'http://data.openstreetmapdata.com/land-polygons-split-4326.zip'], stdout=FNULL, stderr=subprocess.STDOUT) |
||||
|
||||
print('Downloading maps (.osm.pbf, .md5, .poly)') |
||||
first_iteration = True |
||||
for amap in map_list: |
||||
if not first_iteration: |
||||
print(' Sleeping to prevent throttle/blocking') |
||||
time.sleep(30) # Seconds |
||||
else: |
||||
first_iteration = False |
||||
print(' ', end='') |
||||
print(amap) |
||||
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_FILE_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT) |
||||
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_MD5_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT) |
||||
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_POLY_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT) |
||||
|
||||
print('Checking map md5s') |
||||
for amap in map_list: |
||||
map_root = amap.strip().split('/')[-1] |
||||
map_file = os.path.join(cached_maps_dir, map_root + MIRROR_FILE_SUFFIX) |
||||
md5_file = os.path.join(cached_maps_dir, map_root + MIRROR_MD5_SUFFIX) |
||||
print(' ', end='') |
||||
print(amap + ': ', end='') |
||||
print(md5_map.check_hash(map_file, md5_file)) |
||||
|
@ -0,0 +1,32 @@
|
||||
# Copyright 2018 Mike "KemoNine" Crosson |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Borrowed from the web: https://www.joelverhagen.com/blog/2011/02/md5-hash-of-file-in-python/ |
||||
|
||||
import hashlib |
||||
|
||||
def md5Checksum(filePath): |
||||
with open(filePath, 'rb') as fh: |
||||
m = hashlib.md5() |
||||
while True: |
||||
data = fh.read(8192) |
||||
if not data: |
||||
break |
||||
m.update(data) |
||||
return m.hexdigest() |
||||
|
||||
def check_hash(map_file, md5_file): |
||||
published_md5 = None |
||||
with open(md5_file, 'r') as f: |
||||
published_md5 = f.readline().strip().split()[0] |
||||
map_md5 = md5Checksum(map_file) |
||||
return published_md5 == map_md5 |
||||
|
@ -0,0 +1,92 @@
|
||||
# Copyright 2018 Mike "KemoNine" Crosson |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Adapted from https://github.com/mapsforge/mapsforge-creator/blob/master/poly2bb.pl |
||||
|
||||
import re, os |
||||
|
||||
BOUND_BUFFER = 0.1 |
||||
COORD_MATCHER = re.compile('^\s*([0-9.E+-]+)\s+([0-9.E+-]+)\s*$') |
||||
|
||||
def calculate_total_bounds(maps_dir, map_lists): |
||||
cached_maps_dir = os.path.abspath(maps_dir) |
||||
x_min = 360.0 |
||||
x_max = -360.0 |
||||
y_min = 360.0 |
||||
y_max = -360.0 |
||||
for alist in map_lists: |
||||
with open(alist, 'r') as maps: |
||||
for line in maps: |
||||
current_map = line.strip().split('/')[-1] |
||||
bounds = calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly')) |
||||
if bounds['bottom'] < y_min: |
||||
y_min = bounds['bottom'] |
||||
if bounds['left'] < x_min: |
||||
x_min = bounds['left'] |
||||
if bounds['top'] > y_max: |
||||
y_max = bounds['top'] |
||||
if bounds['right'] > x_max: |
||||
x_max = bounds['right'] |
||||
values = {} |
||||
values['bottom'] = y_min |
||||
values['left'] = x_min |
||||
values['top'] = y_max |
||||
values['right'] = x_max |
||||
return values |
||||
|
||||
def calculate_bounds(mappoly): |
||||
# Opposite of max possible values so we find the right bounding box later |
||||
x_min = 360.0 |
||||
x_max = -360.0 |
||||
y_min = 360.0 |
||||
y_max = -360.0 |
||||
|
||||
with open(mappoly, 'r') as polyfile: |
||||
for line in polyfile: |
||||
if COORD_MATCHER.match(line.strip()): |
||||
values = line.split() |
||||
x = float(values[0]) |
||||
y = float(values[1]) |
||||
if x < x_min: |
||||
x_min = x |
||||
if x > x_max: |
||||
x_max = x |
||||
if y < y_min: |
||||
y_min = y |
||||
if y > y_max: |
||||
y_max = y |
||||
|
||||
# Buffer bounds |
||||
x_min = x_min - BOUND_BUFFER |
||||
x_max = x_max + BOUND_BUFFER |
||||
y_min = y_min - BOUND_BUFFER |
||||
y_max = y_max + BOUND_BUFFER |
||||
|
||||
# Cleanup bounding (per original script) |
||||
x_min = -180 if x_min < -180 else x_min |
||||
x_max = 180 if x_max > 180 else x_max |
||||
y_min = -90 if y_min < -90 else y_min |
||||
y_max = 90 if y_max > 90 else y_max |
||||
|
||||
# Dict w/ useful values |
||||
values = {} |
||||
values['bottom'] = y_min |
||||
values['left'] = x_min |
||||
values['top'] = y_max |
||||
values['right'] = x_max |
||||
|
||||
x_center = (x_min + x_max) / 2.0 |
||||
y_center = (y_min + y_max) / 2.0 |
||||
|
||||
values['lat'] = y_center |
||||
values['lon'] = x_center |
||||
|
||||
return values |
@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3 |
||||
# -*- coding: utf-8 -*- |
||||
|
||||
# Copyright 2018 Mike "KemoNine" Crosson |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import subprocess, os, zipfile, shutil, fileinput, glob |
||||
from . import poly |
||||
|
||||
FNULL = open(os.devnull, 'w') |
||||
|
||||
SOURCE_MAP_POSTFIX = '-latest.osm.pbf' |
||||
|
||||
def setup_working_dirs(base_path, current_timestamp, output_map_name): |
||||
print('Creating working directories as needed') |
||||
if not os.path.exists(os.path.join(base_path, 'tmp')): |
||||
os.makedirs(os.path.join(base_path, 'tmp')) |
||||
if not os.path.exists(os.path.join(base_path, 'out')): |
||||
os.makedirs(os.path.join(base_path, 'out')) |
||||
output_dir = current_timestamp + '-' + output_map_name |
||||
if not os.path.exists(os.path.join(base_path, 'out', output_dir)): |
||||
os.makedirs(os.path.join(base_path, 'out', output_dir)) |
||||
|
||||
def prep_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name): |
||||
print('Prepping land/sea polygons (one prep per map in list(s))') |
||||
output_dir = current_timestamp + '-' + output_map_name |
||||
polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys') |
||||
polys_zip = zipfile.ZipFile(os.path.join(maps_dir, 'land-polygons-split-4326.zip')) |
||||
|
||||
for alist in map_lists: |
||||
with open(alist, 'r') as maps: |
||||
for line in maps: |
||||
map_polys_dir = os.path.join(polys_output_dir, line.strip().split('/')[-1]) |
||||
print(' Extracting to: ' + map_polys_dir) |
||||
os.makedirs(map_polys_dir) |
||||
polys_zip.extractall(map_polys_dir) |
||||
|
||||
polys_zip.close() |
||||
|
||||
def process_land_sea_polys(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram): |
||||
print('Processing land/sea polygons (one process per map in list(s))') |
||||
output_dir = current_timestamp + '-' + output_map_name |
||||
polys_output_dir = os.path.join(base_path, 'out', output_dir, 'polys') |
||||
cached_maps_dir = os.path.abspath(maps_dir) |
||||
for alist in map_lists: |
||||
with open(alist, 'r') as maps: |
||||
for line in maps: |
||||
current_map = line.strip().split('/')[-1] |
||||
print(' Processing: ' + current_map) |
||||
bounds = poly.calculate_bounds(os.path.join(cached_maps_dir, current_map + '.poly')) |
||||
subprocess.run(['ogr2ogr', '-overwrite', '-progress', '-skipfailures', '-clipsrc', str(bounds['left']), str(bounds['bottom']), str(bounds['right']), str(bounds['top']), os.path.join(polys_output_dir, current_map + '.shp'), os.path.join(polys_output_dir, current_map, 'land-polygons-split-4326', 'land_polygons.shp')], stdout=FNULL, stderr=subprocess.STDOUT) |
||||
subprocess.run(['python2', os.path.join(base_path, 'shape2osm.py'), '-l', os.path.join(polys_output_dir, current_map), os.path.join(polys_output_dir, current_map + '.shp')], stdout=FNULL, stderr=subprocess.STDOUT) |
||||
shutil.copyfile(os.path.join(base_path, 'sea.osm'), os.path.join(polys_output_dir, current_map + '_sea.osm')) |
||||
with fileinput.FileInput(os.path.join(polys_output_dir, current_map + '_sea.osm'), inplace=True, backup='.bak') as sea: |
||||
for line in sea: |
||||
updated_line = line.replace('$BOTTOM', str(bounds['bottom'])) |
||||
updated_line = updated_line.replace('$LEFT', str(bounds['left'])) |
||||
updated_line = updated_line.replace('$TOP', str(bounds['top'])) |
||||
updated_line = updated_line.replace('$RIGHT', str(bounds['right'])) |
||||
print(updated_line, end='') |
||||
|
||||
map_list = [] |
||||
for alist in map_lists: |
||||
with open(alist, 'r') as maps: |
||||
for line in maps: |
||||
map_list.append(line.strip().split('/')[-1]) |
||||
|
||||
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) |
||||
env = os.environ.copy() |
||||
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) |
||||
|
||||
print('Processing land/sea polys using osmosis') |
||||
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] |
||||
for amap in map_list: |
||||
osmosis_cmd.extend(['--rb', 'file=' + os.path.join(maps_dir, amap + SOURCE_MAP_POSTFIX)]) |
||||
for apolymap in glob.glob(os.path.join(polys_output_dir, amap + '*.osm')): |
||||
print(' Found poly map : ' + apolymap) |
||||
osmosis_cmd.extend(['--rx', 'file=' + apolymap, '--s', '--m']) |
||||
osmosis_cmd.extend(['--log-progress']) |
||||
for x in range(0, len(map_list) - 1): |
||||
osmosis_cmd.append('--merge') |
||||
osmosis_cmd.extend(['--log-progress']) |
||||
osmosis_cmd.extend(['--wb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf']) |
||||
osmosis_cmd.extend(['omitmetadata=true']) |
||||
cmd = subprocess.Popen(osmosis_cmd, env=env) |
||||
cmd.wait() |
||||
|
||||
def osmosis_main(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space, use_ram): |
||||
output_dir = current_timestamp + '-' + output_map_name |
||||
|
||||
bounds = poly.calculate_total_bounds(maps_dir, map_lists) |
||||
|
||||
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) |
||||
env = os.environ.copy() |
||||
env['JAVACMD_OPTIONS'] = '-XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) |
||||
|
||||
print('-----------------------------') |
||||
print('Processing maps using osmosis') |
||||
print('-----------------------------') |
||||
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] |
||||
osmosis_cmd.extend(['--rb', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '_merged_polys.pbf']) |
||||
osmosis_cmd.extend(['--log-progress']) |
||||
osmosis_cmd.extend(['--mapfile-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.map']) |
||||
if use_ram: |
||||
osmosis_cmd.extend(['type=ram']) |
||||
else: |
||||
osmosis_cmd.extend(['type=hd']) |
||||
osmosis_cmd.extend(['bbox=' + str(bounds['bottom']) + ',' + str(bounds['left']) + ',' + str(bounds['top']) + ',' + str(bounds['right'])]) |
||||
osmosis_cmd.extend(['threads=1', 'map-start-zoom=8']) |
||||
cmd = subprocess.Popen(osmosis_cmd, env=env) |
||||
cmd.wait() |
||||
|
||||
def osmosis_pois(base_path, current_timestamp, maps_dir, map_lists, output_map_name, max_heap_space): |
||||
output_dir = current_timestamp + '-' + output_map_name |
||||
|
||||
map_list = [] |
||||
for alist in map_lists: |
||||
with open(alist, 'r') as maps: |
||||
for line in maps: |
||||
map_list.append(os.path.join(maps_dir, line.strip().split('/')[-1]+SOURCE_MAP_POSTFIX)) |
||||
|
||||
# Setup various runtime aspects (going to do multiple osmosis runs (maps AND POIs) |
||||
env = os.environ.copy() |
||||
env['JAVACMD_OPTIONS'] = '-Xmx' + max_heap_space + ' -server -Djava.io.tmpdir=' + os.path.join(base_path, 'tmp') # Setup java temp dir to something a bit more sane (tmpfs /tmp for the loss) |
||||
print('Processing POIs using osmosis') |
||||
osmosis_cmd = [os.path.join(base_path, 'bin', 'osmosis', 'bin', 'osmosis')] |
||||
for amap in map_list: |
||||
osmosis_cmd.extend(['--rb', 'file=' + amap]) |
||||
for x in range(0, len(map_list) - 1): |
||||
osmosis_cmd.append('--merge') |
||||
osmosis_cmd.extend(['--poi-writer', 'file=' + os.path.join(base_path, 'out', output_dir, output_map_name) + '.poi', 'tag-conf-file=' + os.path.join(base_path, 'poi-mapping.xml')]) |
||||
cmd = subprocess.Popen(osmosis_cmd, env=env) |
||||
cmd.wait() |
@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<osm version="0.6"> |
||||
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459320" version="1" lon="$LEFT" lat="$BOTTOM" /> |
||||
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459321" version="1" lon="$LEFT" lat="$TOP" /> |
||||
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459322" version="1" lon="$RIGHT" lat="$TOP" /> |
||||
<node timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951459323" version="1" lon="$RIGHT" lat="$BOTTOM" /> |
||||
<way timestamp="1969-12-31T23:59:59Z" changeset="-1" id="32951623372" version="1"> |
||||
<nd ref="32951459320" /> |
||||
<nd ref="32951459321" /> |
||||
<nd ref="32951459322" /> |
||||
<nd ref="32951459323" /> |
||||
<nd ref="32951459320" /> |
||||
<tag k="area" v="yes" /> |
||||
<tag k="layer" v="-5" /> |
||||
<tag k="natural" v="sea" /> |
||||
</way> |
||||
</osm> |
@ -0,0 +1,434 @@
|
||||
#!/usr/bin/python |
||||
|
||||
""" |
||||
This script is designed to act as assistance in converting shapefiles |
||||
to OpenStreetMap data. This file is optimized and tested with MassGIS |
||||
shapefiles, converted to EPSG:4326 before being passed to the script. |
||||
You can perform this conversion with |
||||
|
||||
ogr2ogr -t_srs EPSG:4326 new_file.shp old_file.shp |
||||
|
||||
It is expected that you will modify the fixed_tags, tag_mapping, and |
||||
boring_tags attributes of this script before running. You should read, |
||||
or at least skim, the code up until it says: |
||||
|
||||
DO NOT CHANGE AFTER THIS LINE. |
||||
|
||||
to accomodate your own data. |
||||
|
||||
Modified to support a maximum waylength and multiple outer ways in a |
||||
multipolygon relationship |
||||
|
||||
adapted by devemux86 |
||||
""" |
||||
|
||||
__author__ = "Christopher Schmidt <crschmidt@crschmidt.net>" |
||||
__version__ = "$Id: polyshp2osm-WaterBody.py,v 1.3 2009/03/30 20:54:13 tbook Exp $" |
||||
|
||||
gdal_install = """ |
||||
Installing GDAL depends on your platform. Information is available at: |
||||
|
||||
http://trac.osgeo.org/gdal/wiki/DownloadingGdalBinaries |
||||
|
||||
For Debian-based systems: |
||||
|
||||
apt-get install python-gdal |
||||
|
||||
will usually suffice. |
||||
""" |
||||
|
||||
# These tags are attached to all exterior ways. You can put any key/value pairs |
||||
# in this dictionary. |
||||
|
||||
fixed_tags = { |
||||
'natural': 'nosea', |
||||
'layer': '-5' |
||||
} |
||||
|
||||
# Here are a number of functions: These functions define tag mappings. The API |
||||
# For these functions is that they are passed the attributes from a feature, |
||||
# and they return a list of two-tuples which match to key/value pairs. |
||||
|
||||
# The following Ftypes are not imported |
||||
# 46100 - Submerged Stream |
||||
ignoreField = "tile_x" |
||||
ignoreValues = [] |
||||
|
||||
# The following Ftypes are imported, but don't have any OSM tags attached: |
||||
# 36100 - Playa - An area from which water evaporates |
||||
|
||||
def ftype(data): |
||||
"""Type of body - From NHD Ftype""" |
||||
natural = { |
||||
'LakePond': 'water', |
||||
'SwampMarsh': 'wetland' |
||||
} |
||||
|
||||
if 'ftype' in data: |
||||
if data['ftype'] in natural: |
||||
return [('natural', natural[data['ftype']])] |
||||
return None |
||||
|
||||
def fcode(data): |
||||
"""For features where the ftype is not specific enough""" |
||||
landuse = { |
||||
'43600': 'reservoir', #Reservoir |
||||
'43601': 'reservoir', #Reservoir |
||||
'43602': 'reservoir', #Reservoir |
||||
'43603': 'reservoir', #Reservoir |
||||
'43604': 'reservoir', #Reservoir |
||||
'43605': 'reservoir', #Reservoir |
||||
'43606': 'reservoir', #Reservoir |
||||
'43607': 'reservoir', #Reservoir |
||||
'43609': 'reservoir', #Reservoir |
||||
'43610': 'reservoir', #Reservoir |
||||
'43611': 'reservoir', #Reservoir |
||||
'43612': 'reservoir', #Reservoir |
||||
'43614': 'reservoir', #Reservoir |
||||
'43615': 'reservoir', #Reservoir |
||||
'43616': 'reservoir', #Reservoir |
||||
'43617': 'reservoir', #Reservoir |
||||
'43618': 'reservoir', #Reservoir |
||||
'43619': 'reservoir', #Reservoir |
||||
'43620': 'reservoir' #Reservoir |
||||
} |
||||
|
||||
manmade = { |
||||
'43613': 'reservoir_covered' #Covered Reservoir |
||||
} |
||||
|
||||
leisure = { |
||||
'43608': 'swimming_pool' #Swimming Pool |
||||
} |
||||
|
||||
if 'fcode' in data: |
||||
keys = [] |
||||
if data['fcode'] in landuse: |
||||
keys.append( ('landuse', landuse[data['fcode']]) ) |
||||
if data['fcode'] in manmade: |
||||
keys.append( ('man_made', manmade[data['fcode']]) ) |
||||
if data['fcode'] in leisure: |
||||
keys.append( ('leisure', leisure[data['fcode']]) ) |
||||
return keys |
||||
return None |
||||
|
||||
# The most important part of the code: define a set of key/value pairs |
||||
# to iterate over to generate keys. This is a list of two-tuples: first |
||||
# is a 'key', which is only used if the second value is a string. In |
||||
# that case, it is a map of lowercased fieldnames to OSM tag names: so |
||||
# fee_owner maps to 'owner' in the OSM output. |
||||
|
||||
# if the latter is callable (has a __call__; is a function), then that |
||||
# method is called, passing in a dict of feature attributes with |
||||
# lowercased key names. Those functions can then return a list of |
||||
# two-tuples to be used as tags, or nothin' to skip the tags. |
||||
|
||||
|
||||
tag_mapping = [ |
||||
('ftype', ftype), |
||||
('fcode', fcode), |
||||
('gnis_name', 'name'), |
||||
('gnis_id', 'gnis:feature_id'), |
||||
('elevation', 'ele'), |
||||
] |
||||
|
||||
# These tags are not exported, even with the source data; this should be |
||||
# used for tags which are usually calculated in a GIS. AREA and LEN are |
||||
# common. |
||||
|
||||
boring_tags = [ 'error', 'tile_x', 'tile_y' ] |
||||
|
||||
# Namespace is used to prefix existing data attributes. If 'None', or |
||||
# '--no-source' is set, then source attributes are not exported, only |
||||
# attributes in tag_mapping. |
||||
|
||||
namespace = "NHD" |
||||
#namespace = None |
||||
|
||||
# Uncomment the "DONT_RUN = False" line to get started. |
||||
|
||||
#DONT_RUN = True |
||||
DONT_RUN = False |
||||
|
||||
# Set the maximum length of a way (in nodes) before it is split into |
||||
# shorter ways |
||||
|
||||
Max_Waylength = 1500000 |
||||
|
||||
# =========== DO NOT CHANGE AFTER THIS LINE. =========================== |
||||
# Below here is regular code, part of the file. This is not designed to |
||||
# be modified by users. |
||||
|