Support reading OSM PBF files

pyproject.toml

@@ -17,19 +17,22 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-    "typing_extensions >=4, <5"
+    "typing_extensions >=4, <5",
+    "protobuf >=5.27, <6",
 ]
 
 [project.urls]
 Home = "https://github.com/MKuranowski/pyroutelib3"
 
 [tool.black]
 line-length = 100
+exclude = "pyroutelib3/osm/pbf"
 
 [tool.isort]
 profile = "black"
 line_length = 100
+skip = ["pyroutelib3/osm/pbf"]
 
 [tool.pyright]
 typeCheckingMode = "strict"
-ignore = ["tests"]
+ignore = ["tests", "pyroutelib3/osm/pbf"]

pyroutelib3/osm/fixtures/simple_graph.osm

@@ -1,55 +1,55 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <osm version='0.6' generator='JOSM'>
-  <node id='-1' lat='-2.73495245962' lon='2.83923666828'>
+  <node id='-1' lat='-2.73495245962' lon='2.83923666828' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-1' />
   </node>
-  <node id='-2' lat='-2.73242793496' lon='2.83923765326'>
+  <node id='-2' lat='-2.73242793496' lon='2.83923765326' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-2' />
   </node>
-  <node id='-61' lat='-2.72972768022' lon='2.83926371472'>
+  <node id='-61' lat='-2.72972768022' lon='2.83926371472' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-61' />
   </node>
-  <node id='-62' lat='-2.72938616409' lon='2.83956383049'>
+  <node id='-62' lat='-2.72938616409' lon='2.83956383049' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-62' />
   </node>
-  <node id='-63' lat='-2.72907392744' lon='2.83920771391'>
+  <node id='-63' lat='-2.72907392744' lon='2.83920771391' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-63' />
   </node>
-  <node id='-60' lat='-2.72941544366' lon='2.83890759814'>
+  <node id='-60' lat='-2.72941544366' lon='2.83890759814' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-60' />
   </node>
-  <node id='-3' lat='-2.73243462926' lon='2.84135576989'>
+  <node id='-3' lat='-2.73243462926' lon='2.84135576989' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-3' />
   </node>
-  <node id='-7' lat='-2.72941089232' lon='2.84139901524'>
+  <node id='-7' lat='-2.72941089232' lon='2.84139901524' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-7' />
   </node>
-  <node id='-8' lat='-2.72825076265' lon='2.84257281775'>
+  <node id='-8' lat='-2.72825076265' lon='2.84257281775' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-8' />
   </node>
-  <node id='-4' lat='-2.73083636925' lon='2.84072562328'>
+  <node id='-4' lat='-2.73083636925' lon='2.84072562328' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-4' />
   </node>
-  <node id='-9' lat='-2.72638097685' lon='2.83923674747'>
+  <node id='-9' lat='-2.72638097685' lon='2.83923674747' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-9' />
   </node>
-  <node id='-5' lat='-2.73091659085' lon='2.84209711884'>
+  <node id='-5' lat='-2.73091659085' lon='2.84209711884' version='1' timestamp='2020-01-01T00:00:00Z'>
     <tag k='ref' v='-5' />
   </node>
-  <way id='-100'>
+  <way id='-100' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-1' />
     <nd ref='-2' />
     <tag k='highway' v='primary' />
     <tag k='ref' v='-100' />
   </way>
-  <way id='-107'>
+  <way id='-107' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-2' />
     <nd ref='-61' />
     <tag k='highway' v='primary' />
     <tag k='motor_vehicle' v='no' />
     <tag k='ref' v='-107' />
   </way>
-  <way id='-108'>
+  <way id='-108' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-63' />
     <nd ref='-60' />
     <nd ref='-61' />
@@ -59,79 +59,79 @@
     <tag k='junction' v='roundabout' />
     <tag k='ref' v='-108' />
   </way>
-  <way id='-101'>
+  <way id='-101' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-2' />
     <nd ref='-3' />
     <tag k='highway' v='unclassified' />
     <tag k='ref' v='-101' />
   </way>
-  <way id='-102'>
+  <way id='-102' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-3' />
     <nd ref='-7' />
     <tag k='highway' v='unclassified' />
     <tag k='ref' v='-102' />
   </way>
-  <way id='-109'>
+  <way id='-109' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-7' />
     <nd ref='-62' />
     <tag k='highway' v='unclassified' />
     <tag k='ref' v='-109' />
   </way>
-  <way id='-110'>
+  <way id='-110' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-8' />
     <nd ref='-7' />
     <tag k='highway' v='unclassified' />
     <tag k='ref' v='-110' />
   </way>
-  <way id='-105'>
+  <way id='-105' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-7' />
     <nd ref='-4' />
     <tag k='highway' v='unclassified' />
     <tag k='oneway' v='yes' />
     <tag k='ref' v='-105' />
   </way>
-  <way id='-103'>
+  <way id='-103' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-4' />
     <nd ref='-3' />
     <tag k='highway' v='motorway' />
     <tag k='ref' v='-103' />
   </way>
-  <way id='-111'>
+  <way id='-111' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-63' />
     <nd ref='-9' />
     <tag k='highway' v='primary' />
     <tag k='ref' v='-111' />
   </way>
-  <way id='-104'>
+  <way id='-104' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-3' />
     <nd ref='-5' />
     <tag k='highway' v='motorway' />
     <tag k='ref' v='-104' />
   </way>
-  <way id='-106'>
+  <way id='-106' version='1' timestamp='2020-01-01T00:00:00Z'>
     <nd ref='-7' />
     <nd ref='-5' />
     <tag k='highway' v='unclassified' />
     <tag k='oneway' v='-1' />
     <tag k='ref' v='-106' />
   </way>
-  <relation id='-200'>
+  <relation id='-200' version='1' timestamp='2020-01-01T00:00:00Z'>
     <member type='way' ref='-110' role='from' />
     <member type='node' ref='-7' role='via' />
     <member type='way' ref='-102' role='to' />
     <tag k='ref' v='-200' />
     <tag k='restriction' v='no_left_turn' />
     <tag k='type' v='restriction' />
   </relation>
-  <relation id='-201'>
+  <relation id='-201' version='1' timestamp='2020-01-01T00:00:00Z'>
     <member type='way' ref='-100' role='from' />
     <member type='node' ref='-2' role='via' />
     <member type='way' ref='-101' role='to' />
     <tag k='ref' v='-201' />
     <tag k='restriction' v='only_right_turn' />
     <tag k='type' v='restriction' />
   </relation>
-  <relation id='-202'>
+  <relation id='-202' version='1' timestamp='2020-01-01T00:00:00Z'>
     <member type='way' ref='-102' role='from' />
     <member type='node' ref='-3' role='via' />
     <member type='way' ref='-104' role='to' />

pyroutelib3/osm/graph.py

@@ -1,10 +1,9 @@
 import sys
 from dataclasses import dataclass, field
 from functools import singledispatchmethod
-from io import DEFAULT_BUFFER_SIZE
 from logging import getLogger
 from math import isfinite
-from typing import IO, Dict, Iterable, List, Literal, Optional, Set, Tuple
+from typing import IO, Dict, Iterable, List, Optional, Set, Tuple
 
 from typing_extensions import Self
 
@@ -129,11 +128,13 @@ def from_file(
         cls,
         profile: Profile,
         buf: IO[bytes],
-        format: Optional[Literal["xml", "bz2", "gz"]] = None,
-        chunk_size: int = DEFAULT_BUFFER_SIZE,
+        format: reader.FILE_FORMAT_T = reader.DEFAULT_FILE_FORMAT,
+        chunk_size: int = reader.DEFAULT_CHUNK_SIZE,
     ) -> Self:
         """Creates a :py:class:`Graph` based on the provided :py:class:`Profile` and features
-        from the provided OSM file.
+        from the provided possibly-compressed
+        `OSM XML <https://wiki.openstreetmap.org/wiki/OSM_XML>`_ or a
+        `OSM PBF <https://wiki.openstreetmap.org/wiki/PBF_Format>`_ file.
 
         ``format`` and ``chunk_size`` are passed through to :py:func:`osm.reader.read_features`.
         """

pyroutelib3/osm/pbf/fileformat_pb2.pyi

@@ -0,0 +1,33 @@
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Optional as _Optional
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class Blob(_message.Message):
+    __slots__ = ("raw_size", "raw", "zlib_data", "lzma_data", "OBSOLETE_bzip2_data", "lz4_data", "zstd_data")
+    RAW_SIZE_FIELD_NUMBER: _ClassVar[int]
+    RAW_FIELD_NUMBER: _ClassVar[int]
+    ZLIB_DATA_FIELD_NUMBER: _ClassVar[int]
+    LZMA_DATA_FIELD_NUMBER: _ClassVar[int]
+    OBSOLETE_BZIP2_DATA_FIELD_NUMBER: _ClassVar[int]
+    LZ4_DATA_FIELD_NUMBER: _ClassVar[int]
+    ZSTD_DATA_FIELD_NUMBER: _ClassVar[int]
+    raw_size: int
+    raw: bytes
+    zlib_data: bytes
+    lzma_data: bytes
+    OBSOLETE_bzip2_data: bytes
+    lz4_data: bytes
+    zstd_data: bytes
+    def __init__(self, raw_size: _Optional[int] = ..., raw: _Optional[bytes] = ..., zlib_data: _Optional[bytes] = ..., lzma_data: _Optional[bytes] = ..., OBSOLETE_bzip2_data: _Optional[bytes] = ..., lz4_data: _Optional[bytes] = ..., zstd_data: _Optional[bytes] = ...) -> None: ...
+
+class BlobHeader(_message.Message):
+    __slots__ = ("type", "indexdata", "datasize")
+    TYPE_FIELD_NUMBER: _ClassVar[int]
+    INDEXDATA_FIELD_NUMBER: _ClassVar[int]
+    DATASIZE_FIELD_NUMBER: _ClassVar[int]
+    type: str
+    indexdata: bytes
+    datasize: int
+    def __init__(self, type: _Optional[str] = ..., indexdata: _Optional[bytes] = ..., datasize: _Optional[int] = ...) -> None: ...