Added Bothies

build/build_bikepack_map.py

@@ -0,0 +1,37 @@
+import os
+import sys
+import logging
+sys.path.append(os.path.abspath('lib'))
+from tools import OSZoomStack, Splitter, Mkgmap, GetMaxOSMID, OSMMerge, OSMSort, GPKG2OSM
+
+extract_bbox = [134340, 20297, 407049, 164276]
+osm_core_file = './build/data/devon-latest.osm.pbf'
+
+output_dir = './build/output'
+splitter_dir = './build/splitter'
+styles_dir = './style/mkgmap_styles'
+typ_file = './style/typ_files/mkgmap-typ-files/bikepack.txt'
+precomp_sea = './build/data/sea/sea-latest.zip'
+bounds = './build/data/bounds/bounds-latest.zip'
+highest_id = 11267580680
+logging.basicConfig(level=logging.INFO)
+
+# Step 1 - Get the Max ID of our OSM data
+mx = GetMaxOSMID()
+if not highest_id:
+    logging.info(f"Getting max ID from {osm_core_file}")
+    mx.apply_file(osm_core_file)
+    logging.info(f'Highest ID = {mx.max_id}')
+
+# Step 2 - Download Zoomstack data and extract contours to OSM format
+
+o = OSZoomStack(bbox=extract_bbox, max_id=mx.max_id)
+file = o.get_zoomstack()
+zoomstack_output = o.make_osm()
+mx.reset()
+mx.apply_file(zoomstack_output)
+logging.info(f'Highest Zoomstack ID = {mx.max_id}')
+
+# Step 3
+
+# STILL in construction

build/test_mkgmap.py

@@ -6,7 +6,7 @@
 
 zoomstack_destination='./build/data/zoomstack'
 extract_bbox = [134340, 20297, 407049, 164276]
-osm_output = './build/data/devon-latest.osm.pbf'
+osm_output = './build/data/great-britain-latest.osm.pbf'
 output_dir = './build/output'
 splitter_dir = './build/splitter'
 styles_dir = './style/mkgmap_styles'
@@ -19,7 +19,7 @@
 # ogr2ogr -f 'GPKG' ./build/data/itts/all-itts-27700-devon.gpkg ./build/data/itts/all-itts-27700.gpkg -spat 134340 20297 407049 164276
 if len(sys.argv) == 1 or sys.argv[1] == 'all':
     # Get the Max ID of the osm data
-
+    print(f'Getting max ID {osm_output}')
     mx = GetMaxOSMID()
     mx.apply_file(osm_output)
     print(f'Highest ID = {mx.max_id}')
@@ -29,7 +29,7 @@
     # destination sets the working file
     # bbox sets the bounding box for the data (27700 CRS)
 
-    o = OSZoomStack(destination=zoomstack_destination, bbox=extract_bbox, max_id=mx.max_id)
+    o = OSZoomStack(destination=zoomstack_destination,  max_id=mx.max_id)
     file = o.get_zoomstack()
     zoomstack_output = o.make_osm()
 
@@ -45,6 +45,7 @@
     merger.merge_all()
 
 if len(sys.argv) > 1 and sys.argv[1] in ('splitter', 'all'):
+    print('Sorting')
 
     # sort by id
     osms = OSMSort(input_file=merged_output, output_file=f'{output_dir}/sorted.pbf')

lib/tools.py

@@ -7,6 +7,7 @@
 import osmnx as ox
 import osmium
 
+
 def executor(params):
     result = subprocess.run(params)
     if result.returncode != 0:
@@ -15,11 +16,12 @@ def executor(params):
     logging.info(f'Ran {params[0]} with {result.stdout}')
     return True
 
+
 class java_runner:
 
     def __init__(self, *args, **kwargs):
         self.params = {
-            key.replace('_','-'):value for key, value in kwargs.items()
+            key.replace('_', '-'): value for key, value in kwargs.items()
         }
         self.params_suffix = None
         self.logging_config = None
@@ -30,20 +32,21 @@ def check(self):
             return False
 
         return True
+
     def run(self):
         if not self.check():
             logging.error('Run checks failed')
             return False
 
         if self.logging_config:
-            command = ['java', f'-Dlog.config={self.logging_config}','-jar', self.jarfile]
+            command = ['java', f'-Dlog.config={self.logging_config}', '-jar', self.jarfile]
         else:
             command = ['java', '-jar', self.jarfile]
 
         for key, value in self.params.items():
             if value and key != 'input-file':
                 command.append(f'--{key}={value}')
-                #command.append(value)
+                # command.append(value)
             else:
                 if key != 'input-file':
                     command.append(f'--{key}')
@@ -55,14 +58,15 @@ def run(self):
             for s in self.params_suffix:
                 command.append(s)
 
-        print(f'Running {command}')
+        logging.info(f'Running {command}')
         result = executor(command)
         return result
 
+
 class GetMaxOSMID(osmium.SimpleHandler):
 
     def __init__(self):
-        super(GetMaxOSMID,self).__init__()
+        super(GetMaxOSMID, self).__init__()
         self.max_id = 0
 
     def node(self, n):
@@ -71,9 +75,10 @@ def node(self, n):
     def reset(self):
         self.max_id = 0
 
+
 class Splitter(java_runner):
     def __init__(self, *args, **kwargs):
-        super().__init__(self,*args,**kwargs)
+        super().__init__(self, *args, **kwargs)
         self.jarfile = kwargs.get('jarfile', 'bin/splitter/splitter.jar')
 
 
@@ -102,13 +107,64 @@ def check(self):
 
         return True
 
-class OSZoomStack:
 
+class GPKG2OSM:
     def __init__(self, *args, **kwargs):
-        self.bbox = kwargs.get('bbox', [271621, 50902, 304818, 68793])
-        self.dest = kwargs.get('destination', './data/zoomstack')
-        self.tag_translator = kwargs.get('tag_translator', 'zoomstack_translator.py')
+        self.bbox = kwargs.get('bbox')
+        self.osm_dest = kwargs.get('destination', './build/output/osm')
+        self.working_files = kwargs.get('working_files', './build/output/working_files')
+        self.tag_translator = kwargs.get('tag_translator', 'bikepack_translator.py')
+        self.clip = kwargs.get('clip', True)
         self.max_id = kwargs.get('max_id', 0)
+        self.file_in = None
+        self.clipped_file = None
+        self.osm_out = None
+
+    def make_osm(self):
+        logging.info(f'Making OSM using bbox {self.bbox if self.bbox is not None else "entire file"}')
+        base_name = os.path.basename(self.file_in)
+        filename_body = os.path.splitext(base_name)[0]
+        self.clipped_file = f'{self.working_files}/{filename_body}_clipped.gpkg'
+        self.osm_out = f'{self.osm_dest}/{filename_body}.osm.pbf'
+
+        response = 'y'
+        if self.clip and self.bbox:
+
+            if os.path.isfile(self.clipped_file):
+                response = input(f'{filename_body} has already been clipped. Do you want to clip and write again? (y/n)')
+
+            if response == 'y':
+                print(f'Clipping  to {self.clipped_file}')
+                ogr_params = ['ogr2ogr', '-f', 'gpkg', '-overwrite', self.clipped_file, self.file_in, '-spat']
+                ogr_params.extend([str(i) for i in self.bbox])
+                executor(ogr_params)
+
+            response = 'y'
+
+        else:
+            self.clipped_file = self.file_in
+
+        if os.path.isfile(self.osm_out):
+            response = input(f'{filename} has already been converted, convert again? (y/n)')
+
+        if response == 'y':
+            logging.info(f'Writing osm data to {self.osm_out}')
+            osm_params = ['ogr2osm', self.clipped_file, '-f', '--pbf', '-o', self.osm_out, '-t', self.tag_translator,
+                          '--id',
+                          str(self.max_id)]
+            executor(osm_params)
+
+        return self.osm_out
+
+
+class OSZoomStack(GPKG2OSM):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(self, *args, **kwargs)
+        self.dest_file = None
+        self.product_list = None
+        self.products = None
+        self.product_filename = None
 
     def get_zoomstack(self, *args, **kwargs):
         self.product_list = OpenDataDownload.all_products()
@@ -129,79 +185,56 @@ def get_zoomstack(self, *args, **kwargs):
 
             response = 'y'
 
-            if os.path.isfile(f'{self.dest}/{self.product_filename}'):
-                logging.info(f'File {self.product_filename} already exists in {self.dest}')
+            if os.path.isfile(f'{self.working_files}/{self.product_filename}'):
+                logging.info(f'File {self.product_filename} already exists in {self.working_files}')
                 response = input('OS Zoomstack has already been downloaded. Do you want to download again? (y/n)')
 
             if response == 'y':
-                logging.info(f'Downloading {self.product_filename} to {self.dest}')
-                self.products.download(file_name=self.product_filename, output_dir=self.dest, overwrite=True)
+                logging.info(f'Downloading {self.product_filename} to {self.working_files}')
+                self.products.download(file_name=self.product_filename, output_dir=self.working_files, overwrite=True)
 
-            self.dest_file = f'{self.dest}/{self.product_filename.replace(".zip", ".gpkg")}'
+            self.dest_file = f'{self.working_files}/{self.product_filename.replace(".zip", ".gpkg")}'
             response = 'y'
             if os.path.isfile(self.dest_file):
-                logging.info(f'File {self.dest_file} already exists in {self.dest}')
+                logging.info(f'File {self.dest_file} already exists in {self.working_files}')
                 response = input('OS Zoomstack has already been unzipped. Do you want to unzip again? (y/n)')
 
             if response == 'y':
-                logging.info(f'Unzipping {self.product_filename} to {self.dest}')
-                with zipfile.ZipFile(f'{self.dest}/{self.product_filename}', 'r') as zip_ref:
-                    zip_ref.extractall(self.dest)
-                    logging.info(f'Extracted {self.product_filename} to {self.dest}')
+                logging.info(f'Unzipping {self.product_filename} to {self.working_files}')
+                with zipfile.ZipFile(f'{self.working_files}/{self.product_filename}', 'r') as zip_ref:
+                    zip_ref.extractall(self.working_files)
+                    logging.info(f'Extracted {self.product_filename} to {self.working_files}')
 
+            self.file_in = self.dest_file
             return self.dest_file
 
-    def make_osm(self, **kwargs):
-        logging.info(f'Making OSM using bbox {self.bbox}')
-        self.osm_in = f'{self.dest}/extract.gpkg'
-        self.osm_out = f'{self.dest}/extract.osm'
-
-        response = 'y'
-        if os.path.isfile(self.dest_file):
-            response = input('OS Zoomstack has already been clipped. Do you want to clip and write again? (y/n)')
-
-        if response == 'y':
-            print(f'Clipping  to {self.dest_file}')
-            ogr_params = ['ogr2ogr', '-f', 'gpkg', '-overwrite', self.osm_in, self.dest_file, '-spat']
-            ogr_params.extend([str(i) for i in self.bbox])
-            executor(ogr_params)
-
-
-        out_file = f'{self.osm_out}.pbf'
-        response = 'y'
-
-        if os.path.isfile(out_file):
-            response = input('OS Zoomstack has already been converted, convert again? (y/n)')
-
-        if response == 'y':
-            print(f'Writing Zoomstack filtered data to {out_file}')
-            osm_params = ['ogr2osm', self.osm_in, '-f', '--pbf', '-o', out_file, '-t', self.tag_translator, '--id', str(self.max_id)]
-            executor(osm_params)
 
-        return out_file
 
 class OSMMerge:
     def __init__(self, **kwargs):
         self.files = kwargs.get('files')
         self.output_file = kwargs.get('output_file')
+
     def merge_all(self):
-        osmium_params = ['osmium','cat']
+        osmium_params = ['osmium', 'cat']
         for f in self.files:
             osmium_params.append(f)
         osmium_params = osmium_params + ['-o', self.output_file, '--overwrite']
         executor(osmium_params)
         return self.output_file
 
+
 class OSMSort:
     def __init__(self, **kwargs):
         self.input_file = kwargs.get('input_file')
         self.output_file = kwargs.get('output_file')
 
     def sort_all(self):
-        osmium_params = ['osmium','sort', '-o', self.output_file, self.input_file, '--overwrite']
+        osmium_params = ['osmium', 'sort', '-o', self.output_file, self.input_file, '--overwrite']
         executor(osmium_params)
         return self.output_file
 
+
 class OSMNXDownloader:
 
     def __init__(self, *args, **kwargs):
@@ -217,10 +250,10 @@ def download(self, *args, **kwargs):
         f = ox.features.features_from_bbox(self.bbox[0], self.bbox[2], self.bbox[1], self.bbox[3], kwargs.get('tags'))
         layer = kwargs.get("file")
         outfile = f'{self.output_dir}/{layer}.geopackage'
-        #f
+        # f
 
-        #list_columns = [col for col in f.columns if f[col].apply(lambda x: isinstance(x, list)).any()]
+        # list_columns = [col for col in f.columns if f[col].apply(lambda x: isinstance(x, list)).any()]
 
         f = f.drop(columns=['nodes'])
         f.to_file(outfile, layer=layer, driver='GPKG')
-        return outfile
+        return outfile

translations/zoomstack_translator.py → translations/bikepack_translator.py

@@ -2,7 +2,7 @@
 from osgeo import ogr
 from layers import layers_to_process
 
-class ZoomStackTranslation(ogr2osm.TranslationBase):
+class BikePackTranslation(ogr2osm.TranslationBase):
 
     def filter_layer(self, layer):
         if not layer:
@@ -34,7 +34,7 @@ def filter_layer(self, layer):
     def filter_tags(self, attrs):
 
         tags = {}
-        tags['source'] = 'oszoomstack'
+        tags['source'] = 'bikepack'
 
         if not attrs:
             return tags