Skip to content

Commit

Permalink
init demoday
Browse files Browse the repository at this point in the history 
* note : rename `vectorfile_format` argument into `file_format` in order to reuse the same functions for dataframe and geodataframe
  • Loading branch information
@tgrandje
tgrandje committed Oct 21, 2023
1 parent 7c5b55f commit b1ee5ea
Show file tree
Hide file tree
Showing 21 changed files with 979 additions and 530 deletions.
6 changes: 3 additions & 3 deletions README.qmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ normandie = cartiflette.s3.download_vectorfile_url_all(
crs = 4326,
values = "11",
borders="COMMUNE",
vectorfile_format="topojson",
file_format="topojson",
filter_by="REGION",
source="EXPRESS-COG-CARTO-TERRITOIRE",
year=2022)
Expand All @@ -57,7 +57,7 @@ regions = cartiflette.s3.download_vectorfile_url_all(
crs = 4326,
values = ["11","27","28"],
borders="COMMUNE",
vectorfile_format="topojson",
file_format="topojson",
filter_by="REGION",
source="EXPRESS-COG-CARTO-TERRITOIRE",
year=2022)
Expand All @@ -71,7 +71,7 @@ regions = cartiflette.s3.download_vectorfile_url_all(
crs = 4326,
values = ["11","27","28"],
borders="COMMUNE_ARRONDISSEMENT",
vectorfile_format="topojson",
file_format="topojson",
filter_by="DEPARTEMENT",
source="EXPRESS-COG-CARTO-TERRITOIRE",
year=2022)
Expand Down
2 changes: 1 addition & 1 deletion cartiflette/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
load_dotenv()

BUCKET = "projet-cartiflette"
PATH_WITHIN_BUCKET = "diffusion/shapefiles-test4"
PATH_WITHIN_BUCKET = "diffusion/shapefiles-test-demoday"
ENDPOINT_URL = "https://minio.lab.sspcloud.fr"

kwargs = {}
Expand Down
19 changes: 15 additions & 4 deletions cartiflette/download/dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,15 +133,26 @@ def _get_last_md5(self) -> None:
self.md5 = md5
except Exception as e:
logger.debug(e)
logger.debug("file not referenced in md5 json")
logger.info(f"{self} not referenced in md5 json")

@pebble.synchronized
def update_json_md5(self, md5: str) -> bool:
"Mise à jour du json des md5"
logger.info(f"updating {self} md5 with {md5} value")
md5 = {
self.provider: {
self.dataset_family: {
self.source: {self.territory: {str(self.year): md5}}
self.provider
if self.provider
else "null": {
self.dataset_family
if self.dataset_family
else "null": {
self.source
if self.source
else "null": {
self.territory
if self.territory
else "null": {str(self.year): md5}
}
}
}
}
Expand Down
14 changes: 7 additions & 7 deletions cartiflette/download/download.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,13 @@ def _upload_raw_dataset_to_s3(
Ex: {
'CHEF_LIEU': [
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
],
'COMMUNE': [
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
],
'ARRONDISSEMENT': [
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
]
}
Expand All @@ -93,7 +93,7 @@ def _upload_raw_dataset_to_s3(
"crs": layer.crs,
"filter_by": None,
"value": None,
"vectorfile_format": layer.format,
"file_format": layer.format,
"provider": layer.provider,
"dataset_family": layer.dataset_family,
"source": layer.source,
Expand Down Expand Up @@ -201,13 +201,13 @@ def _download_sources(
'downloaded': True,
'paths': {
'CHEF_LIEU': [
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
],
'COMMUNE': [
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
],
'ARRONDISSEMENT': [
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
]
}
}
Expand Down
Loading

0 comments on commit b1ee5ea

Please sign in to comment.