# Copyright 2018 Mike "KemoNine" Crosson # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import subprocess, sys, os, pprint, datetime, argparse, time from . import md5_map MIRROR_BASE_URL = 'http://download.geofabrik.de/' MIRROR_FILE_SUFFIX = '-latest.osm.pbf' MIRROR_MD5_SUFFIX = MIRROR_FILE_SUFFIX + '.md5' MIRROR_POLY_SUFFIX = '.poly' def download(dl_dir, dl_lists): FNULL = open(os.devnull, 'w') wget_cmd = 'wget' bunzip2_cmd = 'bunzip2' cached_maps_dir = os.path.abspath(dl_dir) print('Downloading maps to : ' + cached_maps_dir) map_list = [] for alist in dl_lists: with open(alist, 'r') as maps: for line in maps: map_list.append(line.strip()) print('Downloading land polygons from data.openstreetmapdata.com') subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, 'http://data.openstreetmapdata.com/land-polygons-split-4326.zip'], stdout=FNULL, stderr=subprocess.STDOUT) print('Downloading maps (.osm.pbf, .md5, .poly)') first_iteration = True for amap in map_list: if not first_iteration: print(' Sleeping to prevent throttle/blocking') time.sleep(30) # Seconds else: first_iteration = False print(' ', end='') print(amap) subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_FILE_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT) subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_MD5_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT) subprocess.run([wget_cmd, '-nv', '-N', '-P', cached_maps_dir, MIRROR_BASE_URL + amap.strip() + MIRROR_POLY_SUFFIX], stdout=FNULL, stderr=subprocess.STDOUT) print('Checking map md5s') for amap in map_list: map_root = amap.strip().split('/')[-1] map_file = os.path.join(cached_maps_dir, map_root + MIRROR_FILE_SUFFIX) md5_file = os.path.join(cached_maps_dir, map_root + MIRROR_MD5_SUFFIX) print(' ', end='') print(amap + ': ', end='') print(md5_map.check_hash(map_file, md5_file))