From 32564640e7f3b89c6cc946d3e4fe97b261ac88ca Mon Sep 17 00:00:00 2001 From: etj Date: Mon, 22 Apr 2024 17:21:50 +0200 Subject: [PATCH 1/5] Changes to support Assets --- Dockerfile | 1 + importer/handlers/common/raster.py | 4 ++++ importer/handlers/common/vector.py | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index fabb7a98..789aa669 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM geonode/geonode-base:latest-ubuntu-22.04 RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode +RUN cd /usr/src/geonode && git checkout 12124_assets && cd - RUN mkdir -p /usr/src/importer RUN cd .. diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index 7f2377f5..8e51e1e5 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -362,6 +362,10 @@ def create_geonode_resource( dirty_state=True, title=layer_name, owner=_exec.user, + extension=self.supported_file_extension_config["id"], + data_title="Original", + data_type=self.supported_file_extension_config["label"], + link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES files=list( set( list(_exec.input_params.get("files", {}).values()) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index fb198491..eff007d7 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -594,6 +594,10 @@ def create_geonode_resource( dirty_state=True, title=layer_name, owner=_exec.user, + data_title="Original", + data_type=self.supported_file_extension_config["label"], + extension=self.supported_file_extension_config["id"], + link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES files=list( set( list(_exec.input_params.get("files", {}).values()) From e8713cbbbeb6af02a1357a70f46dc0098685304e Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 8 May 2024 11:00:54 +0200 Subject: [PATCH 2/5] Fix asset build --- importer/handlers/common/vector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 95343922..17af9004 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -31,6 +31,7 @@ from importer.api.exception import ImportException from importer.celery_app import importer_app from geonode.storage.manager import storage_manager +from geonode.assets.utils import copy_assets_and_links from importer.handlers.utils import create_alternate, should_be_imported from importer.models import ResourceHandlerInfo @@ -760,14 +761,15 @@ def copy_geonode_resource( new_alternate: str, **kwargs, ): - resource = self.create_geonode_resource( + new_resource = self.create_geonode_resource( layer_name=data_to_update.get("title"), alternate=new_alternate, execution_id=str(_exec.exec_id), - files=resource.files, + files=[], ) - resource.refresh_from_db() - return resource + copy_assets_and_links(resource, target=new_resource) + new_resource.refresh_from_db() + return new_resource def get_ogr2ogr_task_group( self, From e9d8ced5209874f45f0127758f7e78e315a103ef Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 8 May 2024 12:08:52 +0200 Subject: [PATCH 3/5] Fix asset build --- importer/handlers/common/vector.py | 12 +++++++----- importer/tests/unit/test_task.py | 17 ++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 17af9004..345893b6 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -31,7 +31,7 @@ from importer.api.exception import ImportException from importer.celery_app import importer_app from geonode.storage.manager import storage_manager -from geonode.assets.utils import copy_assets_and_links +from geonode.assets.utils import copy_assets_and_links, get_default_asset from importer.handlers.utils import create_alternate, should_be_imported from importer.models import ResourceHandlerInfo @@ -245,10 +245,12 @@ def perform_last_step(execution_id): _exec.save() if _exec and not _exec.input_params.get("store_spatial_file", False): resources = ResourceHandlerInfo.objects.filter(execution_request=_exec) - # getting all files list - resources_files = list(set(chain(*[x.resource.files for x in resources]))) - # better to delete each single file since it can be a remove storage service - list(map(storage_manager.delete, resources_files)) + # getting all assets list + assets = [get_default_asset(x.resource) for x in resources] + # we need to loop and cancel one by one to activate the signal + # which delete the file from the filesystem + for asset in assets: + asset.delete() def extract_resource_to_publish( self, files, action, layer_name, alternate, **kwargs diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index c9b599f9..8ea4c955 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -27,6 +27,7 @@ from dynamic_models.models import ModelSchema, FieldSchema from dynamic_models.exceptions import DynamicModelError, InvalidFieldNameError from importer.models import ResourceHandlerInfo +from importer import project_dir from importer.tests.utils import ( ImporterBaseTestSupport, @@ -39,13 +40,14 @@ class TestCeleryTasks(ImporterBaseTestSupport): def setUp(self): self.user = get_user_model().objects.first() + self.existing_file = f"{project_dir}/tests/fixture/valid.gpkg" self.exec_id = orchestrator.create_execution_request( user=get_user_model().objects.get(username=self.user), func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, # "overwrite_existing_layer": True, "store_spatial_files": True, }, @@ -82,7 +84,7 @@ def test_import_resource_should_rase_exp_if_is_invalid( func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", - input_params={"files": "/filepath", "store_spatial_files": True}, + input_params={"files": self.existing_file, "store_spatial_files": True}, ) is_valid.side_effect = Exception("Invalid format type") @@ -116,7 +118,7 @@ def test_import_resource_should_work( func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", - input_params={"files": "/filepath", "store_spatial_files": True}, + input_params={"files": self.existing_file, "store_spatial_files": True}, ) import_resource( @@ -189,7 +191,7 @@ def test_publish_resource_if_overwrite_should_call_the_publishing( step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, "overwrite_existing_layer": True, "store_spatial_files": True, }, @@ -244,7 +246,7 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing( step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, "overwrite_existing_layer": True, "store_spatial_files": True, }, @@ -388,7 +390,7 @@ def test_rollback_works_as_expected_vector_step( step=conf[0], # step name action="import", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, "overwrite_existing_layer": True, "store_spatial_files": True, "handler_module_path": "importer.handlers.gpkg.handler.GPKGFileHandler", @@ -509,13 +511,14 @@ class TestDynamicModelSchema(TransactionImporterBaseTestSupport): def setUp(self): self.user = get_user_model().objects.first() + self.existing_file = f"{project_dir}/tests/fixture/valid.gpkg" self.exec_id = orchestrator.create_execution_request( user=get_user_model().objects.get(username=self.user), func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, # "overwrite_existing_layer": True, "store_spatial_files": True, }, From 98833fa4e2014829836ff1f0aff5f380c482edc2 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Fri, 10 May 2024 15:51:36 +0200 Subject: [PATCH 4/5] Remove original file at upload end --- importer/handlers/common/metadata.py | 6 ++++++ importer/handlers/common/raster.py | 5 +++++ importer/handlers/common/vector.py | 7 +++++++ 3 files changed, 18 insertions(+) diff --git a/importer/handlers/common/metadata.py b/importer/handlers/common/metadata.py index 5b303577..1ef78fb8 100644 --- a/importer/handlers/common/metadata.py +++ b/importer/handlers/common/metadata.py @@ -7,6 +7,7 @@ from importer.orchestrator import orchestrator from django.shortcuts import get_object_or_404 from geonode.layers.models import Dataset +from geonode.storage.manager import storage_manager logger = logging.getLogger(__name__) @@ -66,6 +67,11 @@ def perform_last_step(execution_id): } ) _exec.save() + # since the original file is now available as asset, we can delete the input files + # TODO must be improved. The asset should be created in the beginning + for _file in _exec.input_params.get("files", {}).values(): + if storage_manager.exists(_file): + storage_manager.delete(_file) def import_resource(self, files: dict, execution_id: str, **kwargs): _exec = orchestrator.get_execution_object(execution_id) diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index 02999693..ce18f156 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -214,6 +214,11 @@ def perform_last_step(execution_id): } ) _exec.save() + # since the original file is now available as asset, we can delete the input files + # TODO must be improved. The asset should be created in the beginning + for _file in _exec.input_params.get("files", {}).values(): + if storage_manager.exists(_file): + storage_manager.delete(_file) def extract_resource_to_publish( self, files, action, layer_name, alternate, **kwargs diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 345893b6..213ff216 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -252,6 +252,13 @@ def perform_last_step(execution_id): for asset in assets: asset.delete() + # since the original file is now available as asset, we can delete the input files + # TODO must be improved. The asset should be created in the beginning + for _file in _exec.input_params.get("files", {}).values(): + if storage_manager.exists(_file): + storage_manager.delete(_file) + + def extract_resource_to_publish( self, files, action, layer_name, alternate, **kwargs ): From 83cd097bb05cfcc6295709a7783367f24b0a5b87 Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Mon, 17 Jun 2024 10:28:45 +0200 Subject: [PATCH 5/5] Assets data retriever (#244) * [Fixes #242] CRS parsing is not correctly handled for CSV files * Let importer create the asset * Add test coverage for asset-importer --- .env_test | 5 ++- Dockerfile | 2 +- importer/__init__.py | 2 +- importer/api/tests.py | 51 ++++++++++++++++++++++++ importer/api/views.py | 42 +++++++++++++++++-- importer/celery_tasks.py | 10 ++++- importer/handlers/README.md | 2 +- importer/handlers/base.py | 7 ---- importer/handlers/common/metadata.py | 1 - importer/handlers/common/raster.py | 22 ++++------ importer/handlers/common/tests_vector.py | 2 +- importer/handlers/common/vector.py | 29 +++++--------- importer/handlers/csv/handler.py | 11 +++-- importer/handlers/sld/tests.py | 5 ++- importer/handlers/xml/tests.py | 5 ++- importer/tests/end2end/test_end2end.py | 44 ++++++++++++++++++++ importer/tests/unit/test_task.py | 18 +++++++-- 17 files changed, 196 insertions(+), 62 deletions(-) diff --git a/.env_test b/.env_test index dcfa53ec..f751770a 100644 --- a/.env_test +++ b/.env_test @@ -178,8 +178,9 @@ DEBUG=False SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' -STATIC_ROOT=/mnt/volumes/statics/static/ -MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +STATIC_ROOT=/tmp/statics/static/ +MEDIA_ROOT=/tmp/statics/uploaded/ +ASSET_ROOT=/tmp/statics/assets/ GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False diff --git a/Dockerfile b/Dockerfile index a6fc2c4a..083ebab5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM geonode/geonode-base:latest-ubuntu-22.04 RUN rm -rf /usr/src/geonode RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode -RUN cd /usr/src/geonode && git checkout 12124_assets && cd - +RUN cd /usr/src/geonode && git fetch --all && git checkout 12124_assets_20240523 && cd - RUN mkdir -p /usr/src/importer RUN cd .. diff --git a/importer/__init__.py b/importer/__init__.py index e3c65827..4138c65c 100644 --- a/importer/__init__.py +++ b/importer/__init__.py @@ -20,7 +20,7 @@ project_dir = os.path.dirname(os.path.abspath(__file__)) -VERSION = (1, 0, 10) +VERSION = (1, 1, 0) __version__ = ".".join([str(i) for i in VERSION]) __author__ = "geosolutions-it" __email__ = "info@geosolutionsgroup.com" diff --git a/importer/api/tests.py b/importer/api/tests.py index bcf8b357..7c54b2ba 100644 --- a/importer/api/tests.py +++ b/importer/api/tests.py @@ -11,6 +11,9 @@ from importer.models import ResourceHandlerInfo from importer.tests.utils import ImporterBaseTestSupport +from importer.orchestrator import orchestrator +from django.utils.module_loading import import_string +from geonode.assets.models import LocalAsset class TestImporterViewSet(ImporterBaseTestSupport): @@ -153,3 +156,51 @@ def test_copy_ther_resource_if_file_handler_is_set(self, _orc): self.assertEqual(200, response.status_code) _orc.s.assert_called_once() + + @patch("importer.api.views.import_orchestrator") + def test_asset_is_created_before_the_import_start(self, patch_upload): + patch_upload.apply_async.side_effect = MagicMock() + + self.client.force_login(get_user_model().objects.get(username="admin")) + payload = { + "base_file": SimpleUploadedFile( + name="test.geojson", content=b"some-content" + ), + "store_spatial_files": True, + } + + response = self.client.post(self.url, data=payload) + + self.assertEqual(201, response.status_code) + + self.assertTrue(201, response.status_code) + + _exec = orchestrator.get_execution_object(response.json()["execution_id"]) + + asset_handler = import_string(_exec.input_params["asset_module_path"]) + self.assertTrue(asset_handler.objects.filter(id=_exec.input_params["asset_id"])) + + asset_handler.objects.filter(id=_exec.input_params["asset_id"]).delete() + + @patch("importer.api.views.import_orchestrator") + @patch( + "importer.api.views.UploadLimitValidator.validate_parallelism_limit_per_user" + ) + def test_asset_should_be_deleted_if_created_during_with_exception( + self, validate_parallelism_limit_per_user, patch_upload + ): + patch_upload.apply_async.s.side_effect = MagicMock() + validate_parallelism_limit_per_user.side_effect = Exception("random exception") + + self.client.force_login(get_user_model().objects.get(username="admin")) + payload = { + "base_file": SimpleUploadedFile( + name="test.geojson", content=b"some-content" + ), + "store_spatial_files": True, + } + + response = self.client.post(self.url, data=payload) + + self.assertEqual(500, response.status_code) + self.assertFalse(LocalAsset.objects.exists()) diff --git a/importer/api/views.py b/importer/api/views.py index 0d71b5eb..6c13157b 100644 --- a/importer/api/views.py +++ b/importer/api/views.py @@ -46,6 +46,8 @@ from rest_framework.parsers import FileUploadParser, MultiPartParser from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.response import Response +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.local import LocalAssetHandler logger = logging.getLogger(__name__) @@ -91,6 +93,8 @@ def create(self, request, *args, **kwargs): """ _file = request.FILES.get("base_file") or request.data.get("base_file") execution_id = None + asset_handler = LocalAssetHandler() + asset_dir = asset_handler._create_asset_dir() serializer = self.get_serializer_class() data = serializer(data=request.data) @@ -111,13 +115,16 @@ def create(self, request, *args, **kwargs): remote_files={"base_file": _data.get("zip_file", _data.get("kmz_file"))} ) # cloning and unzip the base_file - storage_manager.clone_remote_files() + storage_manager.clone_remote_files( + cloning_directory=asset_dir, create_tempdir=False + ) # update the payload with the unziped paths _data.update(storage_manager.get_retrieved_paths()) handler = orchestrator.get_handler(_data) if _file and handler: + asset = None try: # cloning data into a local folder extracted_params, _data = handler.extract_params_from_data(_data) @@ -125,9 +132,13 @@ def create(self, request, *args, **kwargs): # means that the storage manager is not initialized yet, so # the file is not a zip storage_manager = StorageManager(remote_files=_data) - storage_manager.clone_remote_files() + storage_manager.clone_remote_files( + cloning_directory=asset_dir, create_tempdir=False + ) # get filepath - files = storage_manager.get_retrieved_paths() + asset, files = self.generate_asset_and_retrieve_paths( + request, storage_manager, handler + ) upload_validator = UploadLimitValidator(request.user) upload_validator.validate_parallelism_limit_per_user() @@ -144,6 +155,10 @@ def create(self, request, *args, **kwargs): input_params={ **{"files": files, "handler_module_path": str(handler)}, **extracted_params, + **{ + "asset_id": asset.id, + "asset_module_path": f"{asset.__module__}.{asset.__class__.__name__}", + }, }, legacy_upload_name=_file.name, action=action, @@ -159,7 +174,12 @@ def create(self, request, *args, **kwargs): except Exception as e: # in case of any exception, is better to delete the # cloned files to keep the storage under control - if storage_manager is not None: + if asset: + try: + asset.delete() + except Exception as _exc: + logger.warning(_exc) + elif storage_manager is not None: storage_manager.delete_retrieved_paths(force=True) if execution_id: orchestrator.set_as_failed(execution_id=str(execution_id), reason=e) @@ -168,6 +188,20 @@ def create(self, request, *args, **kwargs): raise ImportException(detail="No handlers found for this dataset type") + def generate_asset_and_retrieve_paths(self, request, storage_manager, handler): + asset_handler = asset_handler_registry.get_default_handler() + _files = storage_manager.get_retrieved_paths() + asset = asset_handler.create( + title="Original", + owner=request.user, + description=None, + type=str(handler), + files=list(set(_files.values())), + clone_files=False, + ) + + return asset, _files + class ResourceImporter(DynamicModelViewSet): authentication_classes = [ diff --git a/importer/celery_tasks.py b/importer/celery_tasks.py index c7cfb2f9..a86d3da4 100644 --- a/importer/celery_tasks.py +++ b/importer/celery_tasks.py @@ -329,6 +329,12 @@ def create_geonode_resource( _files = _exec.input_params.get("files") + _asset = ( + import_string(_exec.input_params.get("asset_module_path")) + .objects.filter(id=_exec.input_params.get("asset_id")) + .first() + ) + handler = import_string(handler_module_path)() _overwrite = _exec.input_params.get("overwrite_existing_layer") @@ -337,14 +343,14 @@ def create_geonode_resource( layer_name=layer_name, alternate=alternate, execution_id=execution_id, - files=_files, + asset=_asset, ) else: resource = handler.create_geonode_resource( layer_name=layer_name, alternate=alternate, execution_id=execution_id, - files=_files, + asset=_asset, ) if _overwrite: diff --git a/importer/handlers/README.md b/importer/handlers/README.md index 8916e951..37d982a7 100644 --- a/importer/handlers/README.md +++ b/importer/handlers/README.md @@ -158,7 +158,7 @@ class BaseVectorFileHandler(BaseHandler): return def overwrite_geonode_resource( - self, layer_name: str, alternate: str, execution_id: str, resource_type: Dataset = Dataset, files=None + self, layer_name: str, alternate: str, execution_id: str, resource_type: Dataset = Dataset, asset=None ): """ Base function to override the resource into geonode. Each handler can specify diff --git a/importer/handlers/base.py b/importer/handlers/base.py index 6e29446e..e005fcd4 100644 --- a/importer/handlers/base.py +++ b/importer/handlers/base.py @@ -7,7 +7,6 @@ from importer.utils import ImporterRequestAction as ira from django_celery_results.models import TaskResult from django.db.models import Q -from geonode.storage.manager import storage_manager logger = logging.getLogger(__name__) @@ -150,12 +149,6 @@ def perform_last_step(execution_id): ] _exec.output_params.update({"resources": resource_output_params}) _exec.save() - - # since the original file is now available as asset, we can delete the input files - # TODO must be improved. The asset should be created in the beginning - for _file in _exec.input_params.get("files", {}).values(): - if storage_manager.exists(_file): - storage_manager.delete(_file) return _exec diff --git a/importer/handlers/common/metadata.py b/importer/handlers/common/metadata.py index 8b86db1c..14a80454 100644 --- a/importer/handlers/common/metadata.py +++ b/importer/handlers/common/metadata.py @@ -7,7 +7,6 @@ from importer.orchestrator import orchestrator from django.shortcuts import get_object_or_404 from geonode.layers.models import Dataset -from geonode.storage.manager import storage_manager logger = logging.getLogger(__name__) diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index ae9cd968..2990c4c5 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -312,7 +312,7 @@ def create_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): """ Base function to create the resource into geonode. Each handler can specify @@ -335,6 +335,7 @@ def create_geonode_resource( logger.warning( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) + saved_dataset = resource_manager.create( None, resource_type=resource_type, @@ -346,16 +347,7 @@ def create_geonode_resource( dirty_state=True, title=layer_name, owner=_exec.user, - extension=self.supported_file_extension_config["id"], - data_title="Original", - data_type=self.supported_file_extension_config["label"], - link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES - files=list( - set( - list(_exec.input_params.get("files", {}).values()) - or list(files) - ) - ), + asset=asset, ), ) @@ -377,7 +369,7 @@ def overwrite_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): dataset = resource_type.objects.filter(alternate__icontains=alternate) @@ -405,7 +397,7 @@ def overwrite_geonode_resource( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) return self.create_geonode_resource( - layer_name, alternate, execution_id, resource_type, files + layer_name, alternate, execution_id, resource_type, asset ) elif not dataset.exists() and not _overwrite: logger.warning( @@ -487,9 +479,9 @@ def copy_geonode_resource( layer_name=data_to_update.get("title"), alternate=new_alternate, execution_id=str(_exec.exec_id), - files=kwargs.get("kwargs", {}) + asset=kwargs.get("kwargs", {}) .get("new_file_location", {}) - .get("files", []), + .get("asset", []), ) resource.refresh_from_db() return resource diff --git a/importer/handlers/common/tests_vector.py b/importer/handlers/common/tests_vector.py index a2fe8efb..300b09e9 100644 --- a/importer/handlers/common/tests_vector.py +++ b/importer/handlers/common/tests_vector.py @@ -328,7 +328,7 @@ def test_select_valid_layers(self): self.assertEqual(1, len(valid_layer)) self.assertEqual("mattia_test", valid_layer[0].GetName()) - @override_settings(MEDIA_ROOT='/tmp') + @override_settings(MEDIA_ROOT="/tmp") def test_perform_last_step(self): """ Output params in perform_last_step should return the detail_url and the ID diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index d6b6d3a0..e53a9610 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -2,7 +2,6 @@ from django.db import connections from importer.publisher import DataPublisher from importer.utils import call_rollback_function, find_key_recursively -from itertools import chain import json import logging import os @@ -225,11 +224,10 @@ def perform_last_step(execution_id): # getting all assets list assets = [get_default_asset(x.resource) for x in resources] # we need to loop and cancel one by one to activate the signal - # which delete the file from the filesystem + # that delete the file from the filesystem for asset in assets: asset.delete() - - + def extract_resource_to_publish( self, files, action, layer_name, alternate, **kwargs ): @@ -568,7 +566,7 @@ def create_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): """ Base function to create the resource into geonode. Each handler can specify @@ -591,6 +589,7 @@ def create_geonode_resource( logger.warning( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) + saved_dataset = resource_manager.create( None, resource_type=resource_type, @@ -603,16 +602,7 @@ def create_geonode_resource( dirty_state=True, title=layer_name, owner=_exec.user, - data_title="Original", - data_type=self.supported_file_extension_config["label"], - extension=self.supported_file_extension_config["id"], - link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES - files=list( - set( - list(_exec.input_params.get("files", {}).values()) - or list(files) - ) - ), + asset=asset, ), ) @@ -634,7 +624,7 @@ def overwrite_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): dataset = resource_type.objects.filter(alternate__icontains=alternate) @@ -647,7 +637,7 @@ def overwrite_geonode_resource( dataset = dataset.first() dataset = resource_manager.update( - dataset.uuid, instance=dataset, files=files + dataset.uuid, instance=dataset, files=asset.location ) self.handle_xml_file(dataset, _exec) @@ -663,7 +653,7 @@ def overwrite_geonode_resource( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) return self.create_geonode_resource( - layer_name, alternate, execution_id, resource_type, files + layer_name, alternate, execution_id, resource_type, asset ) elif not dataset.exists() and not _overwrite: logger.warning( @@ -741,11 +731,12 @@ def copy_geonode_resource( new_alternate: str, **kwargs, ): + new_resource = self.create_geonode_resource( layer_name=data_to_update.get("title"), alternate=new_alternate, execution_id=str(_exec.exec_id), - files=[], + asset=get_default_asset(resource), ) copy_assets_and_links(resource, target=new_resource) new_resource.refresh_from_db() diff --git a/importer/handlers/csv/handler.py b/importer/handlers/csv/handler.py index d805883a..f1433ed2 100644 --- a/importer/handlers/csv/handler.py +++ b/importer/handlers/csv/handler.py @@ -243,10 +243,15 @@ def extract_resource_to_publish( return [ { "name": alternate or layer_name, - "crs": ( - self.identify_authority(_l) if _l.GetSpatialRef() else "EPSG:4326" - ), + "crs": (self.identify_authority(_l)), } for _l in layers if self.fixup_name(_l.GetName()) == layer_name ] + + def identify_authority(self, layer): + try: + authority_code = super().identify_authority(layer=layer) + return authority_code + except Exception: + return "EPSG:4326" diff --git a/importer/handlers/sld/tests.py b/importer/handlers/sld/tests.py index ace68be0..26f0eb99 100644 --- a/importer/handlers/sld/tests.py +++ b/importer/handlers/sld/tests.py @@ -24,7 +24,10 @@ def setUpClass(cls): cls.user, _ = get_user_model().objects.get_or_create(username="admin") cls.invalid_files = {"base_file": cls.invalid_sld, "sld_file": cls.invalid_sld} - cls.valid_files = {"base_file": "/tmp/test_sld.sld", "sld_file": "/tmp/test_sld.sld"} + cls.valid_files = { + "base_file": "/tmp/test_sld.sld", + "sld_file": "/tmp/test_sld.sld", + } cls.owner = get_user_model().objects.first() cls.layer = create_single_dataset(name="sld_dataset", owner=cls.owner) diff --git a/importer/handlers/xml/tests.py b/importer/handlers/xml/tests.py index 9262f7c8..8f51e3cc 100644 --- a/importer/handlers/xml/tests.py +++ b/importer/handlers/xml/tests.py @@ -23,7 +23,10 @@ def setUpClass(cls): shutil.copy(cls.valid_xml, "/tmp") cls.user, _ = get_user_model().objects.get_or_create(username="admin") cls.invalid_files = {"base_file": cls.invalid_xml, "xml_file": cls.invalid_xml} - cls.valid_files = {"base_file": "/tmp/test_xml.xml", "xml_file": "/tmp/test_xml.xml"} + cls.valid_files = { + "base_file": "/tmp/test_xml.xml", + "xml_file": "/tmp/test_xml.xml", + } cls.owner = get_user_model().objects.first() cls.layer = create_single_dataset(name="extruded_polygon", owner=cls.owner) diff --git a/importer/tests/end2end/test_end2end.py b/importer/tests/end2end/test_end2end.py index 0cc3001b..dd8de7ab 100644 --- a/importer/tests/end2end/test_end2end.py +++ b/importer/tests/end2end/test_end2end.py @@ -37,6 +37,7 @@ def setUpClass(cls) -> None: } cls.valid_kml = f"{project_dir}/tests/fixture/valid.kml" cls.valid_tif = f"{project_dir}/tests/fixture/test_grid.tif" + cls.valid_csv = f"{project_dir}/tests/fixture/valid.csv" cls.url = reverse("importer_upload") ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] @@ -189,6 +190,7 @@ def test_import_geopackage_with_no_crs_table(self): @mock.patch( "importer.handlers.common.vector.BaseVectorFileHandler._select_valid_layers" ) + @override_settings(MEDIA_ROOT="/tmp/", ASSET_ROOT="/tmp/") def test_import_geopackage_with_no_crs_table_should_raise_error_if_all_layer_are_invalid( self, _select_valid_layers ): @@ -258,6 +260,48 @@ def test_import_geojson_overwrite(self): self.cat.delete(layer) +class ImporterGCSVImportTest(BaseImporterEndToEndTest): + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_geojson(self): + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + + payload = { + "base_file": open(self.valid_csv, "rb"), + } + initial_name = "valid" + self._assertimport(payload, initial_name) + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_csv_overwrite(self): + prev_dataset = create_single_dataset(name="valid") + + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + payload = { + "base_file": open(self.valid_csv, "rb"), + } + initial_name = "valid" + payload["overwrite_existing_layer"] = True + self._assertimport( + payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated + ) + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + + class ImporterKMLImportTest(BaseImporterEndToEndTest): @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) @override_settings( diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index 8d37b219..2bacc12c 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -24,6 +24,7 @@ from geonode.resource.enumerator import ExecutionRequestAction from geonode.base.models import ResourceBase from geonode.base.populate_test_data import create_single_dataset +from geonode.assets.handlers import asset_handler_registry from dynamic_models.models import ModelSchema, FieldSchema from dynamic_models.exceptions import DynamicModelError, InvalidFieldNameError from importer.models import ResourceHandlerInfo @@ -40,7 +41,19 @@ class TestCeleryTasks(ImporterBaseTestSupport): def setUp(self): self.user = get_user_model().objects.first() + self.existing_file = f"{project_dir}/tests/fixture/valid.gpkg" + self.asset_handler = asset_handler_registry.get_default_handler() + + self.asset = self.asset_handler.create( + title="Original", + owner=self.user, + description=None, + type="importer.handlers.gpkg.handler.GPKGFileHandler", + files=[self.existing_file], + clone_files=False, + ) + self.exec_id = orchestrator.create_execution_request( user=get_user_model().objects.get(username=self.user), func_name="dummy_func", @@ -50,6 +63,8 @@ def setUp(self): "files": {"base_file": self.existing_file}, # "overwrite_existing_layer": True, "store_spatial_files": True, + "asset_id": self.asset.id, + "asset_module_path": f"{self.asset.__module__}.{self.asset.__class__.__name__}", }, ) @@ -506,9 +521,6 @@ def test_import_metadata_should_work_as_expected(self): layer.refresh_from_db() self.assertEqual(layer.title, "test_dataset") - #verify that the original has been deleted - self.assertFalse(os.path.exists(xml_in_tmp)) - class TestDynamicModelSchema(TransactionImporterBaseTestSupport): databases = ("default", "datastore")