From 7f3661b1e9ade4bccc57da214e5a9725dcae6c8c Mon Sep 17 00:00:00 2001 From: Mike C Date: Tue, 6 Mar 2018 13:45:10 -0500 Subject: [PATCH] Update readme --- README.md | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 3733bff..37107ea 100644 --- a/README.md +++ b/README.md @@ -50,29 +50,24 @@ optional arguments: ### Examples ``` -#./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt -#./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt -./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name netherlands --map-list lists/netherlands.txt -./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name spain --map-list lists/spain.txt -./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name france --map-list lists/france.txt -./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name belgium --map-list lists/belgium.txt -./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name luxembourg --map-list lists/luxembourg.txt -./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name portugal --map-list lists/portugal.txt -./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia_oceania --map-list lists/australia_oceania.txt +./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name pennsylvania --map-list lists/pennsylvania.txt +./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name idaho --map-list lists/idaho.txt ./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name australia --map-list lists/australia.txt -./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name canada_ontario --map-list lists/ontario.txt -./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name local --map-list lists/midwest.txt --map-list lists/ontario.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_midwest --map-list lists/usa_midwest.txt +./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name canada_ontario --map-list lists/ontario.txt +./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name midwest-2 --map-list lists/midwest.txt --map-list lists/ontario.txt ./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_northeast --map-list lists/usa_northeast.txt ./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_pacific --map-list lists/usa_pacific.txt ./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_south --map-list lists/usa_south.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa_west --map-list lists/usa_west.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/mexico.txt --map-list lists/central_america.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name usa --map-list lists/usa.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name canada --map-list lists/canada.txt -./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name north_america --map-list lists/north_america.txt - +./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name mexico --map-list lists/mexico.txt +./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name mexico_central_america --map-list lists/mexico.txt --map-list lists/central_america.txt +./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name central_america --map-list lists/central_america.txt +./process_maps.py --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name south_america --map-list lists/south_america.txt +./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name netherlands --map-list lists/netherlands.txt +./process_maps.py --use-ram --max-heap-space 24g --maps-dir cache --no-map-download --output-map-name spain --map-list lists/spain.txt +./process_maps.py --max-heap-space 8g --maps-dir cache --no-map-download --output-map-name france --map-list lists/france.txt +./process_maps.py --use-ram --max-heap-space 16g --maps-dir cache --no-map-download --output-map-name belgium --map-list lists/belgium.txt +./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name luxembourg --map-list lists/luxembourg.txt +./process_maps.py --use-ram --max-heap-space 12g --maps-dir cache --no-map-download --output-map-name portugal --map-list lists/portugal.txt ``` # Caching maps for re-use / re-processing @@ -105,6 +100,23 @@ optional arguments: --map-list lists/south_america.txt ``` +# Performance + +This deserves an expansion but for now: notes on how to improve performance on 'small' systems that may not have >64Gb of RAM to work with for larger maps. + +* tmpfs works well for source/tmp/output in disk mode for speedups +* zram /should/ work well if disk mode eats a little too much RAM when backed by tmpfs +* type=ram uses closer to 15-20x source map size in practice +* stacking maps from multiple sources **increases** memory footprint beyond the rough rules of thumb that are called out by the mapsforge devs +* threads=n>1 is a fast way to ruin a heap space (don't do this with type=ram as a general rule of thumb) +* there *are* choke points that are not multi-threaded once the main osmosis 'bits' are done running, deal + +### Look Into + +If you're concerned about Java heap usage, the ZGC announced at the 2018 fosdem conference may be useful. I have no had success building a jre/jdk from sources but you may find the following link useful. + +* https://fosdem.org/2018/schedule/event/zgc/attachments/slides/2211/export/events/attachments/zgc/slides/2211/ZGC_FOSDEM_2018.pdf + # Licencing All code is licensed Apache 2.0 and all non-code is licensed Creative Commons CC-BY-SA-3.0