Skip to content

Commit

Permalink
feat(api) : Switch from using COG to decoupage_administratif
Browse files Browse the repository at this point in the history
  • Loading branch information
vperron committed Sep 12, 2024
1 parent dbe12e1 commit dbf7960
Show file tree
Hide file tree
Showing 22 changed files with 209 additions and 399 deletions.
20 changes: 0 additions & 20 deletions api/requirements/dev-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ argon2-cffi==23.1.0
# via minio
argon2-cffi-bindings==21.2.0
# via argon2-cffi
brotli==1.1.0
# via py7zr
cachetools==5.5.0
# via tox
certifi==2024.8.30
Expand Down Expand Up @@ -101,8 +99,6 @@ idna==3.8
# email-validator
# httpx
# requests
inflate64==1.0.0
# via py7zr
jinja2==3.1.4
# via
# data-inclusion-api (setup.py)
Expand All @@ -115,8 +111,6 @@ markupsafe==2.1.5
# mako
minio==7.2.8
# via data-inclusion-api (setup.py)
multivolumefile==0.2.3
# via py7zr
nodeenv==1.9.1
# via pre-commit
numpy==2.1.1
Expand Down Expand Up @@ -149,26 +143,18 @@ pluggy==1.5.0
# via tox
pre-commit==3.8.0
# via data-inclusion-api (setup.py)
psutil==6.0.0
# via py7zr
psycopg2==2.9.9
# via data-inclusion-api (setup.py)
py7zr==0.22.0
# via data-inclusion-api (setup.py)
pyarrow==17.0.0
# via data-inclusion-api (setup.py)
pyasn1==0.6.1
# via
# python-jose
# rsa
pybcj==1.0.2
# via py7zr
pycparser==2.22
# via cffi
pycryptodome==3.20.0
# via minio
pycryptodomex==3.20.0
# via py7zr
pydantic==2.9.1
# via
# data-inclusion-api (setup.py)
Expand All @@ -190,8 +176,6 @@ pyinstrument==4.7.3
# via fastapi-debug-toolbar
pyogrio==0.9.0
# via geopandas
pyppmd==1.1.0
# via py7zr
pyproj==3.6.1
# via geopandas
pyproject-api==1.7.1
Expand All @@ -215,8 +199,6 @@ pyyaml==6.0.2
# via
# pre-commit
# uvicorn
pyzstd==0.16.1
# via py7zr
requests==2.32.3
# via data-inclusion-api (setup.py)
rsa==4.9
Expand Down Expand Up @@ -246,8 +228,6 @@ sqlparse==0.5.1
# via fastapi-debug-toolbar
starlette==0.38.5
# via fastapi
texttable==1.7.0
# via py7zr
tox==4.18.1
# via data-inclusion-api (setup.py)
tqdm==4.66.5
Expand Down
20 changes: 0 additions & 20 deletions api/requirements/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ argon2-cffi==23.1.0
# via minio
argon2-cffi-bindings==21.2.0
# via argon2-cffi
brotli==1.1.0
# via py7zr
certifi==2024.8.30
# via
# data-inclusion-api (setup.py)
Expand Down Expand Up @@ -81,8 +79,6 @@ idna==3.8
# email-validator
# httpx
# requests
inflate64==1.0.0
# via py7zr
jinja2==3.1.4
# via data-inclusion-api (setup.py)
mako==1.3.5
Expand All @@ -93,8 +89,6 @@ markupsafe==2.1.5
# mako
minio==7.2.8
# via data-inclusion-api (setup.py)
multivolumefile==0.2.3
# via py7zr
numpy==2.1.1
# via
# data-inclusion-api (setup.py)
Expand All @@ -115,26 +109,18 @@ pandas==2.2.2
# via
# data-inclusion-api (setup.py)
# geopandas
psutil==6.0.0
# via py7zr
psycopg2==2.9.9
# via data-inclusion-api (setup.py)
py7zr==0.22.0
# via data-inclusion-api (setup.py)
pyarrow==17.0.0
# via data-inclusion-api (setup.py)
pyasn1==0.6.1
# via
# python-jose
# rsa
pybcj==1.0.2
# via py7zr
pycparser==2.22
# via cffi
pycryptodome==3.20.0
# via minio
pycryptodomex==3.20.0
# via py7zr
pydantic==2.9.1
# via
# data-inclusion-api (setup.py)
Expand All @@ -148,8 +134,6 @@ pydantic-settings==2.5.2
# via data-inclusion-api (setup.py)
pyogrio==0.9.0
# via geopandas
pyppmd==1.1.0
# via py7zr
pyproj==3.6.1
# via geopandas
python-dateutil==2.9.0.post0
Expand All @@ -169,8 +153,6 @@ pytz==2024.2
# pandas
pyyaml==6.0.2
# via uvicorn
pyzstd==0.16.1
# via py7zr
requests==2.32.3
# via data-inclusion-api (setup.py)
rsa==4.9
Expand All @@ -196,8 +178,6 @@ sqlalchemy==2.0.34
# geoalchemy2
starlette==0.38.5
# via fastapi
texttable==1.7.0
# via py7zr
tqdm==4.66.5
# via data-inclusion-api (setup.py)
typing-extensions==4.12.2
Expand Down
20 changes: 0 additions & 20 deletions api/requirements/test-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ argon2-cffi==23.1.0
# via minio
argon2-cffi-bindings==21.2.0
# via argon2-cffi
brotli==1.1.0
# via py7zr
certifi==2024.8.30
# via
# data-inclusion-api (setup.py)
Expand Down Expand Up @@ -89,8 +87,6 @@ idna==3.8
# email-validator
# httpx
# requests
inflate64==1.0.0
# via py7zr
iniconfig==2.0.0
# via pytest
jinja2==3.1.4
Expand All @@ -105,8 +101,6 @@ markupsafe==2.1.5
# mako
minio==7.2.8
# via data-inclusion-api (setup.py)
multivolumefile==0.2.3
# via py7zr
numpy==2.1.1
# via
# data-inclusion-api (setup.py)
Expand All @@ -130,26 +124,18 @@ pandas==2.2.2
# geopandas
pluggy==1.5.0
# via pytest
psutil==6.0.0
# via py7zr
psycopg2==2.9.9
# via data-inclusion-api (setup.py)
py7zr==0.22.0
# via data-inclusion-api (setup.py)
pyarrow==17.0.0
# via data-inclusion-api (setup.py)
pyasn1==0.6.1
# via
# python-jose
# rsa
pybcj==1.0.2
# via py7zr
pycparser==2.22
# via cffi
pycryptodome==3.20.0
# via minio
pycryptodomex==3.20.0
# via py7zr
pydantic==2.9.1
# via
# data-inclusion-api (setup.py)
Expand All @@ -171,8 +157,6 @@ pyinstrument==4.7.3
# via fastapi-debug-toolbar
pyogrio==0.9.0
# via geopandas
pyppmd==1.1.0
# via py7zr
pyproj==3.6.1
# via geopandas
pytest==8.3.3
Expand Down Expand Up @@ -200,8 +184,6 @@ pytz==2024.2
# pandas
pyyaml==6.0.2
# via uvicorn
pyzstd==0.16.1
# via py7zr
requests==2.32.3
# via data-inclusion-api (setup.py)
rsa==4.9
Expand Down Expand Up @@ -231,8 +213,6 @@ starlette==0.38.5
# via fastapi
syrupy==4.7.1
# via data-inclusion-api (setup.py)
texttable==1.7.0
# via py7zr
tqdm==4.66.5
# via data-inclusion-api (setup.py)
typing-extensions==4.12.2
Expand Down
1 change: 0 additions & 1 deletion api/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
"numpy",
"pandas",
"psycopg2",
"py7zr",
"pyarrow",
"pydantic[email]>=2.5.0",
"pydantic-settings",
Expand Down
2 changes: 1 addition & 1 deletion api/src/alembic/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
from alembic import context
from sqlalchemy import engine_from_config, pool

from data_inclusion.api.code_officiel_geo import models as _ # noqa: F401 F811
from data_inclusion.api.config import settings
from data_inclusion.api.core import db
from data_inclusion.api.decoupage_administratif import models as _ # noqa: F401 F811
from data_inclusion.api.inclusion_data import models as _ # noqa: F401 F811
from data_inclusion.api.request import models as _ # noqa: F401 F811

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,8 @@
"""

import sqlalchemy as sa
from alembic import op

from data_inclusion.api.code_officiel_geo import constants
from data_inclusion.api.code_officiel_geo.models import Commune
from data_inclusion.api.inclusion_data.models import Service, Structure

# revision identifiers, used by Alembic.
revision = "9f9a66546e3a"
down_revision = "170af30febde"
Expand All @@ -21,26 +16,6 @@


def upgrade() -> None:
conn = op.get_bind()

# must clean up the data before adding the foreign key
for model in [Structure, Service]:
# remove district codes
for k, v in constants._DISTRICTS_BY_CITY.items():
conn.execute(
sa.update(model)
.where(model.code_insee.startswith(v[0][:3]))
.values({model.code_insee: k})
.returning(1)
)

# remove invalid codes
conn.execute(
sa.update(model)
.where(model.code_insee.not_in(sa.select(Commune.code)))
.values({model.code_insee: None})
)

op.create_foreign_key(
op.f("fk_api__structures__code_insee__api__communes"),
"api__structures",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""Modified api__communes
Revision ID: e3f3dfa4ad01
Revises: 517603187775
Create Date: 2024-08-30 17:58:54.747630
"""

import geoalchemy2
import sqlalchemy as sa
from alembic import op

from data_inclusion.api.core.db import SortedTextArray

# revision identifiers, used by Alembic.
revision = "e3f3dfa4ad01"
down_revision = "517603187775"
branch_labels = None
depends_on = None


def upgrade() -> None:
op.alter_column(
"api__communes", "siren_epci", existing_type=sa.VARCHAR(), nullable=True
)
op.add_column(
"api__communes",
sa.Column(
"codes_postaux",
SortedTextArray(sa.Text()),
nullable=True,
),
)
op.add_column(
"api__communes",
sa.Column(
"centre",
geoalchemy2.types.Geometry(
srid=4326, from_text="ST_GeomFromEWKT", name="geometry"
),
nullable=True,
),
)
op.drop_index("ix_api__communes__geography", table_name="api__communes")
op.drop_column("api__communes", "geom")


def downgrade() -> None:
op.add_column(
"api__communes",
sa.Column(
"geom",
geoalchemy2.types.Geometry(
srid=4326,
spatial_index=False,
from_text="ST_GeomFromEWKT",
name="geometry",
_spatial_index_reflected=True,
),
autoincrement=False,
nullable=True,
),
)
op.create_index(
"ix_api__communes__geography",
"api__communes",
[
sa.text(
"(st_simplify(geom, 0.01::double precision)::geography(Geometry,4326))"
)
],
unique=False,
)
op.drop_column("api__communes", "centre")
op.drop_column("api__communes", "codes_postaux")
op.alter_column(
"api__communes", "siren_epci", existing_type=sa.VARCHAR(), nullable=False
)
Loading

0 comments on commit dbf7960

Please sign in to comment.