diff --git a/.env_test b/.env_test index dcfa53ec..f751770a 100644 --- a/.env_test +++ b/.env_test @@ -178,8 +178,9 @@ DEBUG=False SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' -STATIC_ROOT=/mnt/volumes/statics/static/ -MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +STATIC_ROOT=/tmp/statics/static/ +MEDIA_ROOT=/tmp/statics/uploaded/ +ASSET_ROOT=/tmp/statics/assets/ GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False diff --git a/Dockerfile b/Dockerfile index 7305bdb5..083ebab5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM geonode/geonode-base:latest-ubuntu-22.04 RUN rm -rf /usr/src/geonode RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode +RUN cd /usr/src/geonode && git fetch --all && git checkout 12124_assets_20240523 && cd - RUN mkdir -p /usr/src/importer RUN cd .. diff --git a/importer/__init__.py b/importer/__init__.py index e3c65827..4138c65c 100644 --- a/importer/__init__.py +++ b/importer/__init__.py @@ -20,7 +20,7 @@ project_dir = os.path.dirname(os.path.abspath(__file__)) -VERSION = (1, 0, 10) +VERSION = (1, 1, 0) __version__ = ".".join([str(i) for i in VERSION]) __author__ = "geosolutions-it" __email__ = "info@geosolutionsgroup.com" diff --git a/importer/api/tests.py b/importer/api/tests.py index bcf8b357..7c54b2ba 100644 --- a/importer/api/tests.py +++ b/importer/api/tests.py @@ -11,6 +11,9 @@ from importer.models import ResourceHandlerInfo from importer.tests.utils import ImporterBaseTestSupport +from importer.orchestrator import orchestrator +from django.utils.module_loading import import_string +from geonode.assets.models import LocalAsset class TestImporterViewSet(ImporterBaseTestSupport): @@ -153,3 +156,51 @@ def test_copy_ther_resource_if_file_handler_is_set(self, _orc): self.assertEqual(200, response.status_code) _orc.s.assert_called_once() + + @patch("importer.api.views.import_orchestrator") + def test_asset_is_created_before_the_import_start(self, patch_upload): + patch_upload.apply_async.side_effect = MagicMock() + + self.client.force_login(get_user_model().objects.get(username="admin")) + payload = { + "base_file": SimpleUploadedFile( + name="test.geojson", content=b"some-content" + ), + "store_spatial_files": True, + } + + response = self.client.post(self.url, data=payload) + + self.assertEqual(201, response.status_code) + + self.assertTrue(201, response.status_code) + + _exec = orchestrator.get_execution_object(response.json()["execution_id"]) + + asset_handler = import_string(_exec.input_params["asset_module_path"]) + self.assertTrue(asset_handler.objects.filter(id=_exec.input_params["asset_id"])) + + asset_handler.objects.filter(id=_exec.input_params["asset_id"]).delete() + + @patch("importer.api.views.import_orchestrator") + @patch( + "importer.api.views.UploadLimitValidator.validate_parallelism_limit_per_user" + ) + def test_asset_should_be_deleted_if_created_during_with_exception( + self, validate_parallelism_limit_per_user, patch_upload + ): + patch_upload.apply_async.s.side_effect = MagicMock() + validate_parallelism_limit_per_user.side_effect = Exception("random exception") + + self.client.force_login(get_user_model().objects.get(username="admin")) + payload = { + "base_file": SimpleUploadedFile( + name="test.geojson", content=b"some-content" + ), + "store_spatial_files": True, + } + + response = self.client.post(self.url, data=payload) + + self.assertEqual(500, response.status_code) + self.assertFalse(LocalAsset.objects.exists()) diff --git a/importer/api/views.py b/importer/api/views.py index 0d71b5eb..6c13157b 100644 --- a/importer/api/views.py +++ b/importer/api/views.py @@ -46,6 +46,8 @@ from rest_framework.parsers import FileUploadParser, MultiPartParser from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.response import Response +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.local import LocalAssetHandler logger = logging.getLogger(__name__) @@ -91,6 +93,8 @@ def create(self, request, *args, **kwargs): """ _file = request.FILES.get("base_file") or request.data.get("base_file") execution_id = None + asset_handler = LocalAssetHandler() + asset_dir = asset_handler._create_asset_dir() serializer = self.get_serializer_class() data = serializer(data=request.data) @@ -111,13 +115,16 @@ def create(self, request, *args, **kwargs): remote_files={"base_file": _data.get("zip_file", _data.get("kmz_file"))} ) # cloning and unzip the base_file - storage_manager.clone_remote_files() + storage_manager.clone_remote_files( + cloning_directory=asset_dir, create_tempdir=False + ) # update the payload with the unziped paths _data.update(storage_manager.get_retrieved_paths()) handler = orchestrator.get_handler(_data) if _file and handler: + asset = None try: # cloning data into a local folder extracted_params, _data = handler.extract_params_from_data(_data) @@ -125,9 +132,13 @@ def create(self, request, *args, **kwargs): # means that the storage manager is not initialized yet, so # the file is not a zip storage_manager = StorageManager(remote_files=_data) - storage_manager.clone_remote_files() + storage_manager.clone_remote_files( + cloning_directory=asset_dir, create_tempdir=False + ) # get filepath - files = storage_manager.get_retrieved_paths() + asset, files = self.generate_asset_and_retrieve_paths( + request, storage_manager, handler + ) upload_validator = UploadLimitValidator(request.user) upload_validator.validate_parallelism_limit_per_user() @@ -144,6 +155,10 @@ def create(self, request, *args, **kwargs): input_params={ **{"files": files, "handler_module_path": str(handler)}, **extracted_params, + **{ + "asset_id": asset.id, + "asset_module_path": f"{asset.__module__}.{asset.__class__.__name__}", + }, }, legacy_upload_name=_file.name, action=action, @@ -159,7 +174,12 @@ def create(self, request, *args, **kwargs): except Exception as e: # in case of any exception, is better to delete the # cloned files to keep the storage under control - if storage_manager is not None: + if asset: + try: + asset.delete() + except Exception as _exc: + logger.warning(_exc) + elif storage_manager is not None: storage_manager.delete_retrieved_paths(force=True) if execution_id: orchestrator.set_as_failed(execution_id=str(execution_id), reason=e) @@ -168,6 +188,20 @@ def create(self, request, *args, **kwargs): raise ImportException(detail="No handlers found for this dataset type") + def generate_asset_and_retrieve_paths(self, request, storage_manager, handler): + asset_handler = asset_handler_registry.get_default_handler() + _files = storage_manager.get_retrieved_paths() + asset = asset_handler.create( + title="Original", + owner=request.user, + description=None, + type=str(handler), + files=list(set(_files.values())), + clone_files=False, + ) + + return asset, _files + class ResourceImporter(DynamicModelViewSet): authentication_classes = [ diff --git a/importer/celery_tasks.py b/importer/celery_tasks.py index c7cfb2f9..a86d3da4 100644 --- a/importer/celery_tasks.py +++ b/importer/celery_tasks.py @@ -329,6 +329,12 @@ def create_geonode_resource( _files = _exec.input_params.get("files") + _asset = ( + import_string(_exec.input_params.get("asset_module_path")) + .objects.filter(id=_exec.input_params.get("asset_id")) + .first() + ) + handler = import_string(handler_module_path)() _overwrite = _exec.input_params.get("overwrite_existing_layer") @@ -337,14 +343,14 @@ def create_geonode_resource( layer_name=layer_name, alternate=alternate, execution_id=execution_id, - files=_files, + asset=_asset, ) else: resource = handler.create_geonode_resource( layer_name=layer_name, alternate=alternate, execution_id=execution_id, - files=_files, + asset=_asset, ) if _overwrite: diff --git a/importer/handlers/README.md b/importer/handlers/README.md index 8916e951..37d982a7 100644 --- a/importer/handlers/README.md +++ b/importer/handlers/README.md @@ -158,7 +158,7 @@ class BaseVectorFileHandler(BaseHandler): return def overwrite_geonode_resource( - self, layer_name: str, alternate: str, execution_id: str, resource_type: Dataset = Dataset, files=None + self, layer_name: str, alternate: str, execution_id: str, resource_type: Dataset = Dataset, asset=None ): """ Base function to override the resource into geonode. Each handler can specify diff --git a/importer/handlers/base.py b/importer/handlers/base.py index ea02b786..e005fcd4 100644 --- a/importer/handlers/base.py +++ b/importer/handlers/base.py @@ -149,6 +149,7 @@ def perform_last_step(execution_id): ] _exec.output_params.update({"resources": resource_output_params}) _exec.save() + return _exec def fixup_name(self, name): diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index 00870c57..2990c4c5 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -312,7 +312,7 @@ def create_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): """ Base function to create the resource into geonode. Each handler can specify @@ -335,6 +335,7 @@ def create_geonode_resource( logger.warning( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) + saved_dataset = resource_manager.create( None, resource_type=resource_type, @@ -346,12 +347,7 @@ def create_geonode_resource( dirty_state=True, title=layer_name, owner=_exec.user, - files=list( - set( - list(_exec.input_params.get("files", {}).values()) - or list(files) - ) - ), + asset=asset, ), ) @@ -373,7 +369,7 @@ def overwrite_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): dataset = resource_type.objects.filter(alternate__icontains=alternate) @@ -401,7 +397,7 @@ def overwrite_geonode_resource( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) return self.create_geonode_resource( - layer_name, alternate, execution_id, resource_type, files + layer_name, alternate, execution_id, resource_type, asset ) elif not dataset.exists() and not _overwrite: logger.warning( @@ -483,9 +479,9 @@ def copy_geonode_resource( layer_name=data_to_update.get("title"), alternate=new_alternate, execution_id=str(_exec.exec_id), - files=kwargs.get("kwargs", {}) + asset=kwargs.get("kwargs", {}) .get("new_file_location", {}) - .get("files", []), + .get("asset", []), ) resource.refresh_from_db() return resource diff --git a/importer/handlers/common/tests_vector.py b/importer/handlers/common/tests_vector.py index 2c306812..300b09e9 100644 --- a/importer/handlers/common/tests_vector.py +++ b/importer/handlers/common/tests_vector.py @@ -1,4 +1,5 @@ import os +import shutil import uuid from celery.canvas import Signature from celery import group @@ -13,6 +14,7 @@ from geonode.resource.models import ExecutionRequest from dynamic_models.models import ModelSchema from osgeo import ogr +from django.test.utils import override_settings class TestBaseVectorFileHandler(TestCase): @@ -27,12 +29,16 @@ def setUpClass(cls): cls.no_crs_gpkg = f"{project_dir}/tests/fixture/noCrsTable.gpkg" cls.user, _ = get_user_model().objects.get_or_create(username="admin") cls.invalid_files = {"base_file": cls.invalid_gpkg} - cls.valid_files = {"base_file": cls.valid_gpkg} + cls.valid_files = {"base_file": "/tmp/valid.gpkg"} cls.owner = get_user_model().objects.first() cls.layer = create_single_dataset( name="stazioni_metropolitana", owner=cls.owner ) + def setUp(self) -> None: + shutil.copy(self.valid_gpkg, "/tmp") + super().setUp() + def test_create_error_log(self): """ Should return the formatted way for the log of the handler @@ -322,11 +328,13 @@ def test_select_valid_layers(self): self.assertEqual(1, len(valid_layer)) self.assertEqual("mattia_test", valid_layer[0].GetName()) + @override_settings(MEDIA_ROOT="/tmp") def test_perform_last_step(self): """ Output params in perform_last_step should return the detail_url and the ID of the resource created """ + handler = GPKGFileHandler() # creating exec_id for the import exec_id = orchestrator.create_execution_request( user=get_user_model().objects.first(), @@ -336,15 +344,15 @@ def test_perform_last_step(self): ) # create_geonode_resource - resource = self.handler.create_geonode_resource( + resource = handler.create_geonode_resource( "layer_name", "layer_alternate", str(exec_id), ) exec_obj = orchestrator.get_execution_object(str(exec_id)) - self.handler.create_resourcehandlerinfo(str(self.handler), resource, exec_obj) + handler.create_resourcehandlerinfo(str(handler), resource, exec_obj) # calling the last_step - self.handler.perform_last_step(str(exec_id)) + handler.perform_last_step(str(exec_id)) expected_output = { "resources": [{"id": resource.pk, "detail_url": resource.detail_url}] } diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index dda337fb..e53a9610 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -2,7 +2,6 @@ from django.db import connections from importer.publisher import DataPublisher from importer.utils import call_rollback_function, find_key_recursively -from itertools import chain import json import logging import os @@ -29,7 +28,7 @@ from osgeo import ogr from importer.api.exception import ImportException from importer.celery_app import importer_app -from geonode.storage.manager import storage_manager +from geonode.assets.utils import copy_assets_and_links, get_default_asset from importer.handlers.utils import create_alternate, should_be_imported from importer.models import ResourceHandlerInfo @@ -222,10 +221,12 @@ def perform_last_step(execution_id): _exec = BaseHandler.perform_last_step(execution_id=execution_id) if _exec and not _exec.input_params.get("store_spatial_file", False): resources = ResourceHandlerInfo.objects.filter(execution_request=_exec) - # getting all files list - resources_files = list(set(chain(*[x.resource.files for x in resources]))) - # better to delete each single file since it can be a remove storage service - list(map(storage_manager.delete, resources_files)) + # getting all assets list + assets = [get_default_asset(x.resource) for x in resources] + # we need to loop and cancel one by one to activate the signal + # that delete the file from the filesystem + for asset in assets: + asset.delete() def extract_resource_to_publish( self, files, action, layer_name, alternate, **kwargs @@ -565,7 +566,7 @@ def create_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): """ Base function to create the resource into geonode. Each handler can specify @@ -588,6 +589,7 @@ def create_geonode_resource( logger.warning( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) + saved_dataset = resource_manager.create( None, resource_type=resource_type, @@ -600,12 +602,7 @@ def create_geonode_resource( dirty_state=True, title=layer_name, owner=_exec.user, - files=list( - set( - list(_exec.input_params.get("files", {}).values()) - or list(files) - ) - ), + asset=asset, ), ) @@ -627,7 +624,7 @@ def overwrite_geonode_resource( alternate: str, execution_id: str, resource_type: Dataset = Dataset, - files=None, + asset=None, ): dataset = resource_type.objects.filter(alternate__icontains=alternate) @@ -640,7 +637,7 @@ def overwrite_geonode_resource( dataset = dataset.first() dataset = resource_manager.update( - dataset.uuid, instance=dataset, files=files + dataset.uuid, instance=dataset, files=asset.location ) self.handle_xml_file(dataset, _exec) @@ -656,7 +653,7 @@ def overwrite_geonode_resource( f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created" ) return self.create_geonode_resource( - layer_name, alternate, execution_id, resource_type, files + layer_name, alternate, execution_id, resource_type, asset ) elif not dataset.exists() and not _overwrite: logger.warning( @@ -734,14 +731,16 @@ def copy_geonode_resource( new_alternate: str, **kwargs, ): - resource = self.create_geonode_resource( + + new_resource = self.create_geonode_resource( layer_name=data_to_update.get("title"), alternate=new_alternate, execution_id=str(_exec.exec_id), - files=resource.files, + asset=get_default_asset(resource), ) - resource.refresh_from_db() - return resource + copy_assets_and_links(resource, target=new_resource) + new_resource.refresh_from_db() + return new_resource def get_ogr2ogr_task_group( self, diff --git a/importer/handlers/csv/handler.py b/importer/handlers/csv/handler.py index d805883a..f1433ed2 100644 --- a/importer/handlers/csv/handler.py +++ b/importer/handlers/csv/handler.py @@ -243,10 +243,15 @@ def extract_resource_to_publish( return [ { "name": alternate or layer_name, - "crs": ( - self.identify_authority(_l) if _l.GetSpatialRef() else "EPSG:4326" - ), + "crs": (self.identify_authority(_l)), } for _l in layers if self.fixup_name(_l.GetName()) == layer_name ] + + def identify_authority(self, layer): + try: + authority_code = super().identify_authority(layer=layer) + return authority_code + except Exception: + return "EPSG:4326" diff --git a/importer/handlers/sld/tests.py b/importer/handlers/sld/tests.py index 176ce8a4..26f0eb99 100644 --- a/importer/handlers/sld/tests.py +++ b/importer/handlers/sld/tests.py @@ -1,6 +1,7 @@ +import shutil from django.conf import settings from django.contrib.auth import get_user_model -from django.test import TestCase +from django.test import TestCase, override_settings from geonode.base.populate_test_data import create_single_dataset from importer import project_dir from importer.models import ResourceHandlerInfo @@ -18,12 +19,22 @@ def setUpClass(cls): cls.handler = SLDFileHandler() cls.valid_sld = f"{settings.PROJECT_ROOT}/base/fixtures/test_sld.sld" cls.invalid_sld = f"{project_dir}/tests/fixture/invalid.gpkg" + + shutil.copy(cls.valid_sld, "/tmp") + cls.user, _ = get_user_model().objects.get_or_create(username="admin") cls.invalid_files = {"base_file": cls.invalid_sld, "sld_file": cls.invalid_sld} - cls.valid_files = {"base_file": cls.valid_sld, "sld_file": cls.valid_sld} + cls.valid_files = { + "base_file": "/tmp/test_sld.sld", + "sld_file": "/tmp/test_sld.sld", + } cls.owner = get_user_model().objects.first() cls.layer = create_single_dataset(name="sld_dataset", owner=cls.owner) + def setUp(self) -> None: + shutil.copy(self.valid_sld, "/tmp") + super().setUp() + def test_task_list_is_the_expected_one(self): expected = ( "start_import", @@ -52,7 +63,9 @@ def test_can_handle_should_return_false_for_other_files(self): actual = self.handler.can_handle({"base_file": "random.file"}) self.assertFalse(actual) + @override_settings(MEDIA_ROOT="/tmp/") def test_can_successfully_import_metadata_file(self): + exec_id = orchestrator.create_execution_request( user=get_user_model().objects.first(), func_name="funct1", diff --git a/importer/handlers/xml/handler.py b/importer/handlers/xml/handler.py index 81a9b162..63a74117 100644 --- a/importer/handlers/xml/handler.py +++ b/importer/handlers/xml/handler.py @@ -35,7 +35,6 @@ def is_valid(files, user=None): Define basic validation steps """ # calling base validation checks - try: with open(files.get("base_file")) as _xml: dlxml.fromstring(_xml.read().encode()) diff --git a/importer/handlers/xml/tests.py b/importer/handlers/xml/tests.py index 0ce6a4c6..8f51e3cc 100644 --- a/importer/handlers/xml/tests.py +++ b/importer/handlers/xml/tests.py @@ -1,6 +1,7 @@ +import shutil from django.conf import settings from django.contrib.auth import get_user_model -from django.test import TestCase +from django.test import TestCase, override_settings from geonode.base.populate_test_data import create_single_dataset from importer import project_dir from importer.models import ResourceHandlerInfo @@ -18,12 +19,21 @@ def setUpClass(cls): cls.handler = XMLFileHandler() cls.valid_xml = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" cls.invalid_xml = f"{project_dir}/tests/fixture/invalid.gpkg" + + shutil.copy(cls.valid_xml, "/tmp") cls.user, _ = get_user_model().objects.get_or_create(username="admin") cls.invalid_files = {"base_file": cls.invalid_xml, "xml_file": cls.invalid_xml} - cls.valid_files = {"base_file": cls.valid_xml, "xml_file": cls.valid_xml} + cls.valid_files = { + "base_file": "/tmp/test_xml.xml", + "xml_file": "/tmp/test_xml.xml", + } cls.owner = get_user_model().objects.first() cls.layer = create_single_dataset(name="extruded_polygon", owner=cls.owner) + def setUp(self) -> None: + shutil.copy(self.valid_xml, "/tmp") + super().setUp() + def test_task_list_is_the_expected_one(self): expected = ( "start_import", @@ -52,6 +62,7 @@ def test_can_handle_should_return_false_for_other_files(self): actual = self.handler.can_handle({"base_file": "random.file"}) self.assertFalse(actual) + @override_settings(MEDIA_ROOT="/tmp/") def test_can_successfully_import_metadata_file(self): exec_id = orchestrator.create_execution_request( user=get_user_model().objects.first(), diff --git a/importer/tests/end2end/test_end2end.py b/importer/tests/end2end/test_end2end.py index 0cc3001b..dd8de7ab 100644 --- a/importer/tests/end2end/test_end2end.py +++ b/importer/tests/end2end/test_end2end.py @@ -37,6 +37,7 @@ def setUpClass(cls) -> None: } cls.valid_kml = f"{project_dir}/tests/fixture/valid.kml" cls.valid_tif = f"{project_dir}/tests/fixture/test_grid.tif" + cls.valid_csv = f"{project_dir}/tests/fixture/valid.csv" cls.url = reverse("importer_upload") ogc_server_settings = OGC_Servers_Handler(settings.OGC_SERVER)["default"] @@ -189,6 +190,7 @@ def test_import_geopackage_with_no_crs_table(self): @mock.patch( "importer.handlers.common.vector.BaseVectorFileHandler._select_valid_layers" ) + @override_settings(MEDIA_ROOT="/tmp/", ASSET_ROOT="/tmp/") def test_import_geopackage_with_no_crs_table_should_raise_error_if_all_layer_are_invalid( self, _select_valid_layers ): @@ -258,6 +260,48 @@ def test_import_geojson_overwrite(self): self.cat.delete(layer) +class ImporterGCSVImportTest(BaseImporterEndToEndTest): + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_geojson(self): + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + + payload = { + "base_file": open(self.valid_csv, "rb"), + } + initial_name = "valid" + self._assertimport(payload, initial_name) + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + + @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) + @override_settings( + GEODATABASE_URL=f"{geourl.split('/geonode_data')[0]}/test_geonode_data" + ) + def test_import_csv_overwrite(self): + prev_dataset = create_single_dataset(name="valid") + + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + payload = { + "base_file": open(self.valid_csv, "rb"), + } + initial_name = "valid" + payload["overwrite_existing_layer"] = True + self._assertimport( + payload, initial_name, overwrite=True, last_update=prev_dataset.last_updated + ) + layer = self.cat.get_layer("geonode:valid") + if layer: + self.cat.delete(layer) + + class ImporterKMLImportTest(BaseImporterEndToEndTest): @mock.patch.dict(os.environ, {"GEONODE_GEODATABASE": "test_geonode_data"}) @override_settings( diff --git a/importer/tests/unit/test_orchestrator.py b/importer/tests/unit/test_orchestrator.py index 25f586c1..0a769252 100644 --- a/importer/tests/unit/test_orchestrator.py +++ b/importer/tests/unit/test_orchestrator.py @@ -2,6 +2,7 @@ import uuid from django.conf import settings from django.contrib.auth import get_user_model +from django.test import override_settings from geonode.tests.base import GeoNodeBaseTestSupport from unittest.mock import patch from importer.api.exception import ImportException @@ -125,6 +126,7 @@ def test_perform_next_step(self, mock_celery): mock_celery.assert_called_once() mock_celery.assert_called_with("importer.import_resource") + @override_settings(MEDIA_ROOT="/tmp/") @patch("importer.orchestrator.importer_app.tasks.get") def test_perform_last_import_step(self, mock_celery): # setup test diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index c9b599f9..2bacc12c 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -24,9 +24,11 @@ from geonode.resource.enumerator import ExecutionRequestAction from geonode.base.models import ResourceBase from geonode.base.populate_test_data import create_single_dataset +from geonode.assets.handlers import asset_handler_registry from dynamic_models.models import ModelSchema, FieldSchema from dynamic_models.exceptions import DynamicModelError, InvalidFieldNameError from importer.models import ResourceHandlerInfo +from importer import project_dir from importer.tests.utils import ( ImporterBaseTestSupport, @@ -39,15 +41,30 @@ class TestCeleryTasks(ImporterBaseTestSupport): def setUp(self): self.user = get_user_model().objects.first() + + self.existing_file = f"{project_dir}/tests/fixture/valid.gpkg" + self.asset_handler = asset_handler_registry.get_default_handler() + + self.asset = self.asset_handler.create( + title="Original", + owner=self.user, + description=None, + type="importer.handlers.gpkg.handler.GPKGFileHandler", + files=[self.existing_file], + clone_files=False, + ) + self.exec_id = orchestrator.create_execution_request( user=get_user_model().objects.get(username=self.user), func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, # "overwrite_existing_layer": True, "store_spatial_files": True, + "asset_id": self.asset.id, + "asset_module_path": f"{self.asset.__module__}.{self.asset.__class__.__name__}", }, ) @@ -82,7 +99,7 @@ def test_import_resource_should_rase_exp_if_is_invalid( func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", - input_params={"files": "/filepath", "store_spatial_files": True}, + input_params={"files": self.existing_file, "store_spatial_files": True}, ) is_valid.side_effect = Exception("Invalid format type") @@ -116,7 +133,7 @@ def test_import_resource_should_work( func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", - input_params={"files": "/filepath", "store_spatial_files": True}, + input_params={"files": self.existing_file, "store_spatial_files": True}, ) import_resource( @@ -189,7 +206,7 @@ def test_publish_resource_if_overwrite_should_call_the_publishing( step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, "overwrite_existing_layer": True, "store_spatial_files": True, }, @@ -244,7 +261,7 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing( step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, "overwrite_existing_layer": True, "store_spatial_files": True, }, @@ -388,7 +405,7 @@ def test_rollback_works_as_expected_vector_step( step=conf[0], # step name action="import", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, "overwrite_existing_layer": True, "store_spatial_files": True, "handler_module_path": "importer.handlers.gpkg.handler.GPKGFileHandler", @@ -477,9 +494,10 @@ def test_import_metadata_should_work_as_expected(self): # later will be removed valid_xml = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" shutil.copy(valid_xml, "/tmp") + xml_in_tmp = "/tmp/test_xml.xml" user, _ = get_user_model().objects.get_or_create(username="admin") - valid_files = {"base_file": valid_xml, "xml_file": valid_xml} + valid_files = {"base_file": xml_in_tmp, "xml_file": xml_in_tmp} layer = create_single_dataset("test_dataset_importer") exec_id = orchestrator.create_execution_request( @@ -509,13 +527,14 @@ class TestDynamicModelSchema(TransactionImporterBaseTestSupport): def setUp(self): self.user = get_user_model().objects.first() + self.existing_file = f"{project_dir}/tests/fixture/valid.gpkg" self.exec_id = orchestrator.create_execution_request( user=get_user_model().objects.get(username=self.user), func_name="dummy_func", step="dummy_step", legacy_upload_name="dummy", input_params={ - "files": {"base_file": "/filepath"}, + "files": {"base_file": self.existing_file}, # "overwrite_existing_layer": True, "store_spatial_files": True, },