Updated documentation etc from upstream repo and nyaruka fork

.coveragerc

@@ -0,0 +1,4 @@
+[report]
+omit =
+    */python?.?/*
+    */site-packages/nose/*

.gitignore

@@ -1,3 +1,5 @@
 env
 mapzen_env
-exporter/settings.yaml
+exporter/settings.yaml
+.coverage
+.idea

.travis.yml

@@ -1,8 +1,18 @@
+dist: trusty
 language: python
 python:
   - "2.7"
+install:
+  - sudo apt-get -qq update
+  - sudo apt-get install -qq software-properties-common
+  - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
+  - sudo apt-get update
+  - sudo apt-get install -y libgdal-dev python-dev build-essential libyaml-dev libspatialindex-dev
+  - pip install -r pip-required.txt
+  - pip install -r pip-dev.txt
 
-install: "pip install -r pip-required.txt"
+script:
+  nosetests --with-coverage
 
-# command to run tests, uncomment when test is activated
-#script: nosetests
+after_success:
+  - coveralls

README.md

@@ -1,15 +1,28 @@
 # MapZen Exporter
 
-> Service used to convert and export Admin boundaries such as MapZen Borders
-> into [POSM Extracts](https://github.com/nyaruka/posm-extracts) expected format.
+[![Build Status](https://travis-ci.org/eHealthAfrica/mapzen-exporter.svg?branch=master)](https://travis-ci.org/eHealthAfrica/mapzen-exporter) [![Coverage Status](https://coveralls.io/repos/eHealthAfrica/mapzen-exporter/badge.svg?branch=master&service=github)](https://coveralls.io/github/eHealthAfrica/mapzen-exporter?branch=master)
+
+> This a tool that adapted and simplified [POSM-Extractor](https://github.com/nyaruka/posm) which is used to parse and export Admin boundaries from OSM data files such as .pbf, .05m using
+> [Fences Builder](https://github.com/pelias/fences-builder)
+> into [POSM Extracts](https://github.com/nyaruka/posm-extracts) similar format.
 
 ### Setup Guide
 
 You need to do the following:
 
+####  Installation of GDAL
+```
+$ sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
+$ sudo apt-get update
+$ sudo apt-get install -y libgdal-dev python-dev build-essential libyaml-dev libspatialindex-dev
+$ sudo apt-get install nodejs-legacy # needed on ubuntu 14.04
+```
+
 ```sh
     $ git clone https://github.com/ehealthafrica/mapzen-exporter && cd mapzen-exporter
 
+    $ npm install -g fences-builder
+
     $ virtualenv mapzen_env
 
     $ source mapzen_env/bin/activate
@@ -25,8 +38,9 @@ You need to do the following:
 - In settings.yaml set the following:
     - `data_dir` : full path to the input geojson files directory
     - `output_dir`: full path to output directory
-    - `mapzen_file_url`: url to online archived file that holds each country's admin boundaries files
-    - `geojson_dir_name`: the path to directory that contains extracted admin boundaries geojson files
+    - `data_url`: url to OSM data file server
+    - `input_dir`: the path to directory that fences-builder will extract admin levels from OSM data file into.
+    - country_osm_id: OSM id of the country you want to extract from the data file.
     - admin_levels:
       > `- 2` # Default is 2 for all countries
 
@@ -39,5 +53,29 @@ You need to do the following:
 - To run, while in exporter directory
 
 ```sh
-    $ python exproter.py
- ```
+    $ python exporter.py run_all
+ ```
+
+### OSM Data Sources:
+  Ensure that the region selected on the map covers the country you want to extract.
+
+  Caveat: for GeoFabrik, Country OSM data file does not extract country boundary use continent's data file or use
+  [BBBike](http://extract.bbbike.org/) and make sure the region selected for extraction covers the country you want
+  to extract its admin levels.
+
+- [GeoFabrik](http://download.geofabrik.de/africa.html)
+- [BBBike](http://extract.bbbike.org/)
+
+### Test
+```sh
+    $ nosetests --with-coverage
+ ```
+
+### Utilities
+
+- `python exporter.py init_dir` : Initialises empty data directories.
+- `python exporter.py download_OSM` : Downloads given OSM data file from remote server.
+- `python exporter.py extract_admins` : Extracts OSM data file admin levels as geojson into given input directory.
+- `python exporter.py generate_output` : Generates a given country's admin levels and write resulting files to output
+directory
+- `python exporter.py run_all`: Runs all the above task in one step.

exporter/exporter.py

@@ -2,36 +2,117 @@
 # -*- coding: utf-8 -*-
 
 import argparse
+import wgetter
 
 from settings.mapzen_settings import MapZenSettings
 from utility import utils
 from parser import process
+import os
 
 argparser = argparse.ArgumentParser(
-    description='Exports given MapZen extracts .zip file'
+    description='Exports given OSM data file admin levels for given country id'
 )
 
 argparser.add_argument(
     '--settings', default='settings.yaml',
     help='path to the settings file, default: settings.yaml'
 )
 
-if __name__ == '__main__':
-    args = argparser.parse_args()
+sub_tasks = argparser.add_subparsers(title='Tasks')
+
+# Initialize folder task
+def init_dir(args):
+    config = MapZenSettings(args.settings)
+    settings = config.get_settings()
+    dest = settings.get('sources').get('data_dir')
+    input_dir = settings.get('sources').get('input_dir')
+    output_dir = settings.get('sources').get('output_dir')
+
+    utils.init_path(dest)
+    utils.init_path(output_dir)
+    utils.init_path(input_dir)
+
+
+task_init_dir = sub_tasks.add_parser(
+    'init_dir', help='Initializes data directory'
+)
+task_init_dir.set_defaults(func=init_dir)
+
+
+# Download OSM data file task
+def download_OSM(args):
     config = MapZenSettings(args.settings)
     settings = config.get_settings()
+    dest = settings.get('sources').get('data_dir')
+    url = settings.get('sources').get('data_url')
+    filename = settings.get('sources').get('filename')
+
+    file_path = wgetter.download(url, outdir=dest)
+    data_file = os.path.join(dest, filename)
+    if file_path:
+        os.rename(file_path, data_file)
 
-    url = settings.get('sources').get('mapzen_file_url')
+
+task_download_osm = sub_tasks.add_parser(
+    'download_OSM', help='Downloads OSM data file from given URl'
+)
+task_download_osm.set_defaults(func=download_OSM)
+
+
+# Extract geojson from OSM data file using Fences Builder
+def extract_admins(args):
+    config = MapZenSettings(args.settings)
+    settings = config.get_settings()
+    input_dir = settings.get('sources').get('input_dir')
     dest = settings.get('sources').get('data_dir')
-    extracted_geojson_dir = settings.get('sources').get('geojson_dir_name')
+    filename = settings.get('sources').get('filename')
+    data_file = os.path.join(dest, filename)
+    cmd = 'fences-builder --inputFile={in_file} --outputDir={out_dir}'.format(in_file=data_file, out_dir=input_dir)
+
+    result = os.system(cmd)
+    if result is not 0:
+        raise SystemExit('Fence Builder extraction of OSM data file failed with error code: ', result)
+
+
+task_extract_admin = sub_tasks.add_parser(
+    'extract_admins', help='Extract admin levels from given OSM data file'
+)
+task_extract_admin.set_defaults(func=extract_admins)
+
+
+# Parse and export a given country geojson
+def generate_output(args):
+    config = MapZenSettings(args.settings)
+    settings = config.get_settings()
+    input_dir = settings.get('sources').get('input_dir')
     output_dir = settings.get('sources').get('output_dir')
+    country_id = settings.get('country_osm_id')
     admin_levels = sorted(settings.get('admin_levels'))
     tolerance = settings.get('tolerance')
 
-    # set up data file, extract file and
-    utils.init_path(dest)
-    utils.init_path(output_dir)
-    file_path = utils.download(url, dest)
-    utils.extract(file_path, dest)
+    process.parse(input_dir, output_dir, admin_levels, tolerance, country_id)
+
+
+task_generate_output = sub_tasks.add_parser(
+    'generate_output', help='Read admin level geojsons from input dir and export'
+)
+task_generate_output.set_defaults(func=generate_output)
+
+
+# Run all task from start to end
+def run_all(args):
+    init_dir(args)
+    download_OSM(args)
+    extract_admins(args)
+    generate_output(args)
+
+task_run_all = sub_tasks.add_parser(
+    'run_all', help='Run all subtasks from start to finish'
+)
+task_run_all.set_defaults(func=run_all)
+
+
+if __name__ == '__main__':
+    args = argparser.parse_args()
+    args.func(args)
 
-    process.parse(extracted_geojson_dir, output_dir, admin_levels, tolerance)