feat(all) : Récupérer les données géo depuis l'API découpage administratif

api/requirements/dev-requirements.txt

@@ -14,8 +14,6 @@ argon2-cffi==23.1.0
     # via minio
 argon2-cffi-bindings==21.2.0
     # via argon2-cffi
-brotli==1.1.0
-    # via py7zr
 cachetools==5.5.0
     # via tox
 certifi==2024.8.30
@@ -101,8 +99,6 @@ idna==3.8
     #   email-validator
     #   httpx
     #   requests
-inflate64==1.0.0
-    # via py7zr
 jinja2==3.1.4
     # via
     #   data-inclusion-api (setup.py)
@@ -115,8 +111,6 @@ markupsafe==2.1.5
     #   mako
 minio==7.2.8
     # via data-inclusion-api (setup.py)
-multivolumefile==0.2.3
-    # via py7zr
 nodeenv==1.9.1
     # via pre-commit
 numpy==2.1.1
@@ -149,26 +143,18 @@ pluggy==1.5.0
     # via tox
 pre-commit==3.8.0
     # via data-inclusion-api (setup.py)
-psutil==6.0.0
-    # via py7zr
 psycopg2==2.9.9
     # via data-inclusion-api (setup.py)
-py7zr==0.22.0
-    # via data-inclusion-api (setup.py)
 pyarrow==17.0.0
     # via data-inclusion-api (setup.py)
 pyasn1==0.6.1
     # via
     #   python-jose
     #   rsa
-pybcj==1.0.2
-    # via py7zr
 pycparser==2.22
     # via cffi
 pycryptodome==3.20.0
     # via minio
-pycryptodomex==3.20.0
-    # via py7zr
 pydantic==2.9.1
     # via
     #   data-inclusion-api (setup.py)
@@ -190,8 +176,6 @@ pyinstrument==4.7.3
     # via fastapi-debug-toolbar
 pyogrio==0.9.0
     # via geopandas
-pyppmd==1.1.0
-    # via py7zr
 pyproj==3.6.1
     # via geopandas
 pyproject-api==1.7.1
@@ -215,8 +199,6 @@ pyyaml==6.0.2
     # via
     #   pre-commit
     #   uvicorn
-pyzstd==0.16.1
-    # via py7zr
 requests==2.32.3
     # via data-inclusion-api (setup.py)
 rsa==4.9
@@ -246,8 +228,6 @@ sqlparse==0.5.1
     # via fastapi-debug-toolbar
 starlette==0.38.5
     # via fastapi
-texttable==1.7.0
-    # via py7zr
 tox==4.18.1
     # via data-inclusion-api (setup.py)
 tqdm==4.66.5

api/requirements/requirements.txt

@@ -13,8 +13,6 @@ argon2-cffi==23.1.0
     # via minio
 argon2-cffi-bindings==21.2.0
     # via argon2-cffi
-brotli==1.1.0
-    # via py7zr
 certifi==2024.8.30
     # via
     #   data-inclusion-api (setup.py)
@@ -81,8 +79,6 @@ idna==3.8
     #   email-validator
     #   httpx
     #   requests
-inflate64==1.0.0
-    # via py7zr
 jinja2==3.1.4
     # via data-inclusion-api (setup.py)
 mako==1.3.5
@@ -93,8 +89,6 @@ markupsafe==2.1.5
     #   mako
 minio==7.2.8
     # via data-inclusion-api (setup.py)
-multivolumefile==0.2.3
-    # via py7zr
 numpy==2.1.1
     # via
     #   data-inclusion-api (setup.py)
@@ -115,26 +109,18 @@ pandas==2.2.2
     # via
     #   data-inclusion-api (setup.py)
     #   geopandas
-psutil==6.0.0
-    # via py7zr
 psycopg2==2.9.9
     # via data-inclusion-api (setup.py)
-py7zr==0.22.0
-    # via data-inclusion-api (setup.py)
 pyarrow==17.0.0
     # via data-inclusion-api (setup.py)
 pyasn1==0.6.1
     # via
     #   python-jose
     #   rsa
-pybcj==1.0.2
-    # via py7zr
 pycparser==2.22
     # via cffi
 pycryptodome==3.20.0
     # via minio
-pycryptodomex==3.20.0
-    # via py7zr
 pydantic==2.9.1
     # via
     #   data-inclusion-api (setup.py)
@@ -148,8 +134,6 @@ pydantic-settings==2.5.2
     # via data-inclusion-api (setup.py)
 pyogrio==0.9.0
     # via geopandas
-pyppmd==1.1.0
-    # via py7zr
 pyproj==3.6.1
     # via geopandas
 python-dateutil==2.9.0.post0
@@ -169,8 +153,6 @@ pytz==2024.2
     #   pandas
 pyyaml==6.0.2
     # via uvicorn
-pyzstd==0.16.1
-    # via py7zr
 requests==2.32.3
     # via data-inclusion-api (setup.py)
 rsa==4.9
@@ -196,8 +178,6 @@ sqlalchemy==2.0.34
     #   geoalchemy2
 starlette==0.38.5
     # via fastapi
-texttable==1.7.0
-    # via py7zr
 tqdm==4.66.5
     # via data-inclusion-api (setup.py)
 typing-extensions==4.12.2

api/requirements/test-requirements.txt

@@ -14,8 +14,6 @@ argon2-cffi==23.1.0
     # via minio
 argon2-cffi-bindings==21.2.0
     # via argon2-cffi
-brotli==1.1.0
-    # via py7zr
 certifi==2024.8.30
     # via
     #   data-inclusion-api (setup.py)
@@ -89,8 +87,6 @@ idna==3.8
     #   email-validator
     #   httpx
     #   requests
-inflate64==1.0.0
-    # via py7zr
 iniconfig==2.0.0
     # via pytest
 jinja2==3.1.4
@@ -105,8 +101,6 @@ markupsafe==2.1.5
     #   mako
 minio==7.2.8
     # via data-inclusion-api (setup.py)
-multivolumefile==0.2.3
-    # via py7zr
 numpy==2.1.1
     # via
     #   data-inclusion-api (setup.py)
@@ -130,26 +124,18 @@ pandas==2.2.2
     #   geopandas
 pluggy==1.5.0
     # via pytest
-psutil==6.0.0
-    # via py7zr
 psycopg2==2.9.9
     # via data-inclusion-api (setup.py)
-py7zr==0.22.0
-    # via data-inclusion-api (setup.py)
 pyarrow==17.0.0
     # via data-inclusion-api (setup.py)
 pyasn1==0.6.1
     # via
     #   python-jose
     #   rsa
-pybcj==1.0.2
-    # via py7zr
 pycparser==2.22
     # via cffi
 pycryptodome==3.20.0
     # via minio
-pycryptodomex==3.20.0
-    # via py7zr
 pydantic==2.9.1
     # via
     #   data-inclusion-api (setup.py)
@@ -171,8 +157,6 @@ pyinstrument==4.7.3
     # via fastapi-debug-toolbar
 pyogrio==0.9.0
     # via geopandas
-pyppmd==1.1.0
-    # via py7zr
 pyproj==3.6.1
     # via geopandas
 pytest==8.3.3
@@ -200,8 +184,6 @@ pytz==2024.2
     #   pandas
 pyyaml==6.0.2
     # via uvicorn
-pyzstd==0.16.1
-    # via py7zr
 requests==2.32.3
     # via data-inclusion-api (setup.py)
 rsa==4.9
@@ -231,8 +213,6 @@ starlette==0.38.5
     # via fastapi
 syrupy==4.7.1
     # via data-inclusion-api (setup.py)
-texttable==1.7.0
-    # via py7zr
 tqdm==4.66.5
     # via data-inclusion-api (setup.py)
 typing-extensions==4.12.2

api/setup.py

@@ -27,7 +27,6 @@
         "numpy",
         "pandas",
         "psycopg2",
-        "py7zr",
         "pyarrow",
         "pydantic[email]>=2.5.0",
         "pydantic-settings",

api/src/alembic/env.py

@@ -3,9 +3,9 @@
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 
-from data_inclusion.api.code_officiel_geo import models as _  # noqa: F401 F811
 from data_inclusion.api.config import settings
 from data_inclusion.api.core import db
+from data_inclusion.api.decoupage_administratif import models as _  # noqa: F401 F811
 from data_inclusion.api.inclusion_data import models as _  # noqa: F401 F811
 from data_inclusion.api.request import models as _  # noqa: F401 F811
 

api/src/alembic/versions/20240527_130609_9f9a66546e3a_add_fk_structure_commune.py

@@ -6,13 +6,8 @@
 
 """
 
-import sqlalchemy as sa
 from alembic import op
 
-from data_inclusion.api.code_officiel_geo import constants
-from data_inclusion.api.code_officiel_geo.models import Commune
-from data_inclusion.api.inclusion_data.models import Service, Structure
-
 # revision identifiers, used by Alembic.
 revision = "9f9a66546e3a"
 down_revision = "170af30febde"
@@ -21,26 +16,6 @@
 
 
 def upgrade() -> None:
-    conn = op.get_bind()
-
-    # must clean up the data before adding the foreign key
-    for model in [Structure, Service]:
-        # remove district codes
-        for k, v in constants._DISTRICTS_BY_CITY.items():
-            conn.execute(
-                sa.update(model)
-                .where(model.code_insee.startswith(v[0][:3]))
-                .values({model.code_insee: k})
-                .returning(1)
-            )
-
-        # remove invalid codes
-        conn.execute(
-            sa.update(model)
-            .where(model.code_insee.not_in(sa.select(Commune.code)))
-            .values({model.code_insee: None})
-        )
-
     op.create_foreign_key(
         op.f("fk_api__structures__code_insee__api__communes"),
         "api__structures",

api/src/alembic/versions/20240830_175854_e3f3dfa4ad01_modified_api__communes.py

@@ -0,0 +1,78 @@
+"""Modified api__communes
+
+Revision ID: e3f3dfa4ad01
+Revises: 517603187775
+Create Date: 2024-08-30 17:58:54.747630
+
+"""
+
+import geoalchemy2
+import sqlalchemy as sa
+from alembic import op
+
+from data_inclusion.api.core.db import SortedTextArray
+
+# revision identifiers, used by Alembic.
+revision = "e3f3dfa4ad01"
+down_revision = "517603187775"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "api__communes", "siren_epci", existing_type=sa.VARCHAR(), nullable=True
+    )
+    op.add_column(
+        "api__communes",
+        sa.Column(
+            "codes_postaux",
+            SortedTextArray(sa.Text()),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "api__communes",
+        sa.Column(
+            "centre",
+            geoalchemy2.types.Geometry(
+                srid=4326, from_text="ST_GeomFromEWKT", name="geometry"
+            ),
+            nullable=True,
+        ),
+    )
+    op.drop_index("ix_api__communes__geography", table_name="api__communes")
+    op.drop_column("api__communes", "geom")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "api__communes",
+        sa.Column(
+            "geom",
+            geoalchemy2.types.Geometry(
+                srid=4326,
+                spatial_index=False,
+                from_text="ST_GeomFromEWKT",
+                name="geometry",
+                _spatial_index_reflected=True,
+            ),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )
+    op.create_index(
+        "ix_api__communes__geography",
+        "api__communes",
+        [
+            sa.text(
+                "(st_simplify(geom, 0.01::double precision)::geography(Geometry,4326))"
+            )
+        ],
+        unique=False,
+    )
+    op.drop_column("api__communes", "centre")
+    op.drop_column("api__communes", "codes_postaux")
+    op.alter_column(
+        "api__communes", "siren_epci", existing_type=sa.VARCHAR(), nullable=False
+    )