diff --git a/build/build_bikepack_map.py b/build/build_bikepack_map.py new file mode 100644 index 0000000..a2b5104 --- /dev/null +++ b/build/build_bikepack_map.py @@ -0,0 +1,37 @@ +import os +import sys +import logging +sys.path.append(os.path.abspath('lib')) +from tools import OSZoomStack, Splitter, Mkgmap, GetMaxOSMID, OSMMerge, OSMSort, GPKG2OSM + +extract_bbox = [134340, 20297, 407049, 164276] +osm_core_file = './build/data/devon-latest.osm.pbf' + +output_dir = './build/output' +splitter_dir = './build/splitter' +styles_dir = './style/mkgmap_styles' +typ_file = './style/typ_files/mkgmap-typ-files/bikepack.txt' +precomp_sea = './build/data/sea/sea-latest.zip' +bounds = './build/data/bounds/bounds-latest.zip' +highest_id = 11267580680 +logging.basicConfig(level=logging.INFO) + +# Step 1 - Get the Max ID of our OSM data +mx = GetMaxOSMID() +if not highest_id: + logging.info(f"Getting max ID from {osm_core_file}") + mx.apply_file(osm_core_file) + logging.info(f'Highest ID = {mx.max_id}') + +# Step 2 - Download Zoomstack data and extract contours to OSM format + +o = OSZoomStack(bbox=extract_bbox, max_id=mx.max_id) +file = o.get_zoomstack() +zoomstack_output = o.make_osm() +mx.reset() +mx.apply_file(zoomstack_output) +logging.info(f'Highest Zoomstack ID = {mx.max_id}') + +# Step 3 + +# STILL in construction diff --git a/build/data/itts/bothies.gpkg b/build/data/bothies/bothies.gpkg similarity index 100% rename from build/data/itts/bothies.gpkg rename to build/data/bothies/bothies.gpkg diff --git a/build/test_mkgmap.py b/build/test_mkgmap.py index 63cb871..9ea719c 100644 --- a/build/test_mkgmap.py +++ b/build/test_mkgmap.py @@ -6,7 +6,7 @@ zoomstack_destination='./build/data/zoomstack' extract_bbox = [134340, 20297, 407049, 164276] -osm_output = './build/data/devon-latest.osm.pbf' +osm_output = './build/data/great-britain-latest.osm.pbf' output_dir = './build/output' splitter_dir = './build/splitter' styles_dir = './style/mkgmap_styles' @@ -19,7 +19,7 @@ # ogr2ogr -f 'GPKG' ./build/data/itts/all-itts-27700-devon.gpkg ./build/data/itts/all-itts-27700.gpkg -spat 134340 20297 407049 164276 if len(sys.argv) == 1 or sys.argv[1] == 'all': # Get the Max ID of the osm data - + print(f'Getting max ID {osm_output}') mx = GetMaxOSMID() mx.apply_file(osm_output) print(f'Highest ID = {mx.max_id}') @@ -29,7 +29,7 @@ # destination sets the working file # bbox sets the bounding box for the data (27700 CRS) - o = OSZoomStack(destination=zoomstack_destination, bbox=extract_bbox, max_id=mx.max_id) + o = OSZoomStack(destination=zoomstack_destination, max_id=mx.max_id) file = o.get_zoomstack() zoomstack_output = o.make_osm() @@ -45,6 +45,7 @@ merger.merge_all() if len(sys.argv) > 1 and sys.argv[1] in ('splitter', 'all'): + print('Sorting') # sort by id osms = OSMSort(input_file=merged_output, output_file=f'{output_dir}/sorted.pbf') diff --git a/lib/tools.py b/lib/tools.py index bfeca73..dc7adf8 100644 --- a/lib/tools.py +++ b/lib/tools.py @@ -7,6 +7,7 @@ import osmnx as ox import osmium + def executor(params): result = subprocess.run(params) if result.returncode != 0: @@ -15,11 +16,12 @@ def executor(params): logging.info(f'Ran {params[0]} with {result.stdout}') return True + class java_runner: def __init__(self, *args, **kwargs): self.params = { - key.replace('_','-'):value for key, value in kwargs.items() + key.replace('_', '-'): value for key, value in kwargs.items() } self.params_suffix = None self.logging_config = None @@ -30,20 +32,21 @@ def check(self): return False return True + def run(self): if not self.check(): logging.error('Run checks failed') return False if self.logging_config: - command = ['java', f'-Dlog.config={self.logging_config}','-jar', self.jarfile] + command = ['java', f'-Dlog.config={self.logging_config}', '-jar', self.jarfile] else: command = ['java', '-jar', self.jarfile] for key, value in self.params.items(): if value and key != 'input-file': command.append(f'--{key}={value}') - #command.append(value) + # command.append(value) else: if key != 'input-file': command.append(f'--{key}') @@ -55,14 +58,15 @@ def run(self): for s in self.params_suffix: command.append(s) - print(f'Running {command}') + logging.info(f'Running {command}') result = executor(command) return result + class GetMaxOSMID(osmium.SimpleHandler): def __init__(self): - super(GetMaxOSMID,self).__init__() + super(GetMaxOSMID, self).__init__() self.max_id = 0 def node(self, n): @@ -71,9 +75,10 @@ def node(self, n): def reset(self): self.max_id = 0 + class Splitter(java_runner): def __init__(self, *args, **kwargs): - super().__init__(self,*args,**kwargs) + super().__init__(self, *args, **kwargs) self.jarfile = kwargs.get('jarfile', 'bin/splitter/splitter.jar') @@ -102,13 +107,64 @@ def check(self): return True -class OSZoomStack: +class GPKG2OSM: def __init__(self, *args, **kwargs): - self.bbox = kwargs.get('bbox', [271621, 50902, 304818, 68793]) - self.dest = kwargs.get('destination', './data/zoomstack') - self.tag_translator = kwargs.get('tag_translator', 'zoomstack_translator.py') + self.bbox = kwargs.get('bbox') + self.osm_dest = kwargs.get('destination', './build/output/osm') + self.working_files = kwargs.get('working_files', './build/output/working_files') + self.tag_translator = kwargs.get('tag_translator', 'bikepack_translator.py') + self.clip = kwargs.get('clip', True) self.max_id = kwargs.get('max_id', 0) + self.file_in = None + self.clipped_file = None + self.osm_out = None + + def make_osm(self): + logging.info(f'Making OSM using bbox {self.bbox if self.bbox is not None else "entire file"}') + base_name = os.path.basename(self.file_in) + filename_body = os.path.splitext(base_name)[0] + self.clipped_file = f'{self.working_files}/{filename_body}_clipped.gpkg' + self.osm_out = f'{self.osm_dest}/{filename_body}.osm.pbf' + + response = 'y' + if self.clip and self.bbox: + + if os.path.isfile(self.clipped_file): + response = input(f'{filename_body} has already been clipped. Do you want to clip and write again? (y/n)') + + if response == 'y': + print(f'Clipping to {self.clipped_file}') + ogr_params = ['ogr2ogr', '-f', 'gpkg', '-overwrite', self.clipped_file, self.file_in, '-spat'] + ogr_params.extend([str(i) for i in self.bbox]) + executor(ogr_params) + + response = 'y' + + else: + self.clipped_file = self.file_in + + if os.path.isfile(self.osm_out): + response = input(f'{filename} has already been converted, convert again? (y/n)') + + if response == 'y': + logging.info(f'Writing osm data to {self.osm_out}') + osm_params = ['ogr2osm', self.clipped_file, '-f', '--pbf', '-o', self.osm_out, '-t', self.tag_translator, + '--id', + str(self.max_id)] + executor(osm_params) + + return self.osm_out + + +class OSZoomStack(GPKG2OSM): + + def __init__(self, *args, **kwargs): + super().__init__(self, *args, **kwargs) + self.dest_file = None + self.product_list = None + self.products = None + self.product_filename = None def get_zoomstack(self, *args, **kwargs): self.product_list = OpenDataDownload.all_products() @@ -129,79 +185,56 @@ def get_zoomstack(self, *args, **kwargs): response = 'y' - if os.path.isfile(f'{self.dest}/{self.product_filename}'): - logging.info(f'File {self.product_filename} already exists in {self.dest}') + if os.path.isfile(f'{self.working_files}/{self.product_filename}'): + logging.info(f'File {self.product_filename} already exists in {self.working_files}') response = input('OS Zoomstack has already been downloaded. Do you want to download again? (y/n)') if response == 'y': - logging.info(f'Downloading {self.product_filename} to {self.dest}') - self.products.download(file_name=self.product_filename, output_dir=self.dest, overwrite=True) + logging.info(f'Downloading {self.product_filename} to {self.working_files}') + self.products.download(file_name=self.product_filename, output_dir=self.working_files, overwrite=True) - self.dest_file = f'{self.dest}/{self.product_filename.replace(".zip", ".gpkg")}' + self.dest_file = f'{self.working_files}/{self.product_filename.replace(".zip", ".gpkg")}' response = 'y' if os.path.isfile(self.dest_file): - logging.info(f'File {self.dest_file} already exists in {self.dest}') + logging.info(f'File {self.dest_file} already exists in {self.working_files}') response = input('OS Zoomstack has already been unzipped. Do you want to unzip again? (y/n)') if response == 'y': - logging.info(f'Unzipping {self.product_filename} to {self.dest}') - with zipfile.ZipFile(f'{self.dest}/{self.product_filename}', 'r') as zip_ref: - zip_ref.extractall(self.dest) - logging.info(f'Extracted {self.product_filename} to {self.dest}') + logging.info(f'Unzipping {self.product_filename} to {self.working_files}') + with zipfile.ZipFile(f'{self.working_files}/{self.product_filename}', 'r') as zip_ref: + zip_ref.extractall(self.working_files) + logging.info(f'Extracted {self.product_filename} to {self.working_files}') + self.file_in = self.dest_file return self.dest_file - def make_osm(self, **kwargs): - logging.info(f'Making OSM using bbox {self.bbox}') - self.osm_in = f'{self.dest}/extract.gpkg' - self.osm_out = f'{self.dest}/extract.osm' - - response = 'y' - if os.path.isfile(self.dest_file): - response = input('OS Zoomstack has already been clipped. Do you want to clip and write again? (y/n)') - - if response == 'y': - print(f'Clipping to {self.dest_file}') - ogr_params = ['ogr2ogr', '-f', 'gpkg', '-overwrite', self.osm_in, self.dest_file, '-spat'] - ogr_params.extend([str(i) for i in self.bbox]) - executor(ogr_params) - - - out_file = f'{self.osm_out}.pbf' - response = 'y' - - if os.path.isfile(out_file): - response = input('OS Zoomstack has already been converted, convert again? (y/n)') - - if response == 'y': - print(f'Writing Zoomstack filtered data to {out_file}') - osm_params = ['ogr2osm', self.osm_in, '-f', '--pbf', '-o', out_file, '-t', self.tag_translator, '--id', str(self.max_id)] - executor(osm_params) - return out_file class OSMMerge: def __init__(self, **kwargs): self.files = kwargs.get('files') self.output_file = kwargs.get('output_file') + def merge_all(self): - osmium_params = ['osmium','cat'] + osmium_params = ['osmium', 'cat'] for f in self.files: osmium_params.append(f) osmium_params = osmium_params + ['-o', self.output_file, '--overwrite'] executor(osmium_params) return self.output_file + class OSMSort: def __init__(self, **kwargs): self.input_file = kwargs.get('input_file') self.output_file = kwargs.get('output_file') def sort_all(self): - osmium_params = ['osmium','sort', '-o', self.output_file, self.input_file, '--overwrite'] + osmium_params = ['osmium', 'sort', '-o', self.output_file, self.input_file, '--overwrite'] executor(osmium_params) return self.output_file + class OSMNXDownloader: def __init__(self, *args, **kwargs): @@ -217,10 +250,10 @@ def download(self, *args, **kwargs): f = ox.features.features_from_bbox(self.bbox[0], self.bbox[2], self.bbox[1], self.bbox[3], kwargs.get('tags')) layer = kwargs.get("file") outfile = f'{self.output_dir}/{layer}.geopackage' - #f + # f - #list_columns = [col for col in f.columns if f[col].apply(lambda x: isinstance(x, list)).any()] + # list_columns = [col for col in f.columns if f[col].apply(lambda x: isinstance(x, list)).any()] f = f.drop(columns=['nodes']) f.to_file(outfile, layer=layer, driver='GPKG') - return outfile \ No newline at end of file + return outfile diff --git a/translations/zoomstack_translator.py b/translations/bikepack_translator.py similarity index 98% rename from translations/zoomstack_translator.py rename to translations/bikepack_translator.py index cdbeda8..018ec5c 100644 --- a/translations/zoomstack_translator.py +++ b/translations/bikepack_translator.py @@ -2,7 +2,7 @@ from osgeo import ogr from layers import layers_to_process -class ZoomStackTranslation(ogr2osm.TranslationBase): +class BikePackTranslation(ogr2osm.TranslationBase): def filter_layer(self, layer): if not layer: @@ -34,7 +34,7 @@ def filter_layer(self, layer): def filter_tags(self, attrs): tags = {} - tags['source'] = 'oszoomstack' + tags['source'] = 'bikepack' if not attrs: return tags