64 lines
2.4 KiB
Python
64 lines
2.4 KiB
Python
|
# Copyright 2018 Mike "KemoNine" Crosson
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
|
||
|
|
import subprocess, sys, os, pprint, datetime, argparse, time
|
||
|
|
from . import md5_map
|
||
|
|
|
||
|
|
MIRROR_BASE_URL = 'http://download.geofabrik.de/'
|
||
|
|
MIRROR_FILE_SUFFIX = '-latest.osm.pbf'
|
||
|
|
MIRROR_MD5_SUFFIX = MIRROR_FILE_SUFFIX + '.md5'
|
||
|
|
MIRROR_POLY_SUFFIX = '.poly'
|
||
|
|
|
||
|
|
def download(dl_dir, dl_lists):
|
||
|
|
|
||
|
|
FNULL = open(os.devnull, 'w')
|
||
|
|
|
||
|
|
wget_cmd = 'wget'
|
||
|
|
bunzip2_cmd = 'bunzip2'
|
||
|
|
|
||
|
|
cached_maps_dir = os.path.abspath(dl_dir)
|
||
|
|
|
||
|
|
print('Downloading maps to : ' + cached_maps_dir)
|
||
|
|
|
||
|
|
map_list = []
|
||
|
|
|
||
|
|
for alist in dl_lists:
|
||
|
|
with open(alist, 'r') as maps:
|
||
|
|
for line in maps:
|
||
|
|
map_list.append(line.strip())
|
||
|
|
|
||
|
|
print('Downloading land polygons from data.openstreetmapdata.com')
|
||
|
|
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, 'http://data.openstreetmapdata.com/land-polygons-split-4326.zip'], stdout=FNULL, stderr=subprocess.STDOUT)
|
||
|
|
|
||
|
|
print('Downloading maps (.osm.pbf, .md5, .poly)')
|
||
|
|
first_iteration = True
|
||
|
|
for amap in map_list:
|
||
|
|
if not first_iteration:
|
||
|
|
print(' Sleeping to prevent throttle/blocking')
|
||
|
|
time.sleep(30) # Seconds
|
||
|
|
else:
|
||
|
|
first_iteration = False
|
||
|
|
print(' ', end='')
|
||
|
|
print(amap)
|
||
|
|
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_FILE_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
|
||
|
|
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_MD5_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
|
||
|
|
subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_POLY_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT)
|
||
|
|
|
||
|
|
print('Checking map md5s')
|
||
|
|
for amap in map_list:
|
||
|
|
map_root = amap.strip().split('/')[-1]
|
||
|
|
map_file = os.path.join(cached_maps_dir, map_root + MIRROR_FILE_SUFFIX)
|
||
|
|
md5_file = os.path.join(cached_maps_dir, map_root + MIRROR_MD5_SUFFIX)
|
||
|
|
print(' ', end='')
|
||
|
|
print(amap + ': ', end='')
|
||
|
|
print(md5_map.check_hash(map_file, md5_file))
|
||
|
|
|