From a7338461268e077c2091a8b0fddf42695ddd266c Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Fri, 13 Oct 2023 16:33:58 +0200 Subject: [PATCH 1/2] BACKPORT #11421 DownloadHandler implementation --- geonode/base/api/serializers.py | 46 ++++++- geonode/base/api/tests.py | 53 ++++++++ geonode/documents/models.py | 8 ++ geonode/documents/tests.py | 22 +++- geonode/geoserver/tests/test_server.py | 25 ++++ geonode/layers/api/tests.py | 24 ++++ geonode/layers/download_handler.py | 168 +++++++++++++++++++++++++ geonode/layers/models.py | 2 - geonode/layers/tests.py | 74 +++++++++-- geonode/layers/utils.py | 35 ++++++ geonode/layers/views.py | 82 ++---------- geonode/security/permissions.py | 6 +- geonode/security/tests.py | 5 +- geonode/settings.py | 6 +- geonode/utils.py | 29 ----- 15 files changed, 462 insertions(+), 123 deletions(-) create mode 100644 geonode/layers/download_handler.py diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 4d07b9478f7..5dff12ccc55 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -30,6 +30,8 @@ from rest_framework import serializers from rest_framework_gis import fields from rest_framework.reverse import reverse, NoReverseMatch +from deprecated import deprecated +from geonode.layers.utils import get_dataset_download_handlers, get_default_dataset_download_handler from dynamic_rest.serializers import DynamicEphemeralSerializer, DynamicModelSerializer from dynamic_rest.fields.fields import DynamicRelationField, DynamicComputedField @@ -53,7 +55,7 @@ from geonode.groups.models import GroupCategory, GroupProfile from geonode.utils import build_absolute_uri -from geonode.security.utils import get_resources_with_perms +from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes from geonode.resource.models import ExecutionRequest import logging @@ -278,8 +280,12 @@ class DownloadLinkField(DynamicComputedField): def __init__(self, **kwargs): super().__init__(**kwargs) + @deprecated(version="4.2.0", reason="Will be replaced by download_urls") def get_attribute(self, instance): try: + logger.info( + "This field is deprecated, and will be removed in the future GeoNode version. Please refer to download_urls" + ) _instance = instance.get_real_instance() return _instance.download_url if hasattr(_instance, "download_url") else None except Exception as e: @@ -287,6 +293,43 @@ def get_attribute(self, instance): return None +class DownloadArrayLinkField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + try: + _instance = instance.get_real_instance() + except Exception as e: + logger.exception(e) + raise e + if _instance.resource_type in ["map"] + get_geoapp_subtypes(): + return [] + elif _instance.resource_type in ["document"]: + return [ + { + "url": _instance.download_url, + "ajax_safe": _instance.download_is_ajax_safe, + } + ] + elif _instance.resource_type in ["dataset"]: + download_urls = [] + # lets get only the default one first to set it + default_handler = get_default_dataset_download_handler() + obj = default_handler(self.context.get("request"), _instance.alternate) + if obj.download_url: + download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": True}) + # then let's prepare the payload with everything + handler_list = get_dataset_download_handlers() + for handler in handler_list: + obj = handler(self.context.get("request"), _instance.alternate) + if obj.download_url: + download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": False}) + return download_urls + else: + return [] + + class FavoriteField(DynamicComputedField): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -481,6 +524,7 @@ def __init__(self, *args, **kwargs): self.fields["is_copyable"] = serializers.BooleanField(read_only=True) self.fields["download_url"] = DownloadLinkField(read_only=True) + self.fields["download_urls"] = DownloadArrayLinkField(read_only=True) self.fields["favorite"] = FavoriteField(read_only=True) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 5a278f432bd..4805cf7b12b 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -2464,6 +2464,59 @@ def test_user_without_view_perms_cannot_see_the_endpoint(self): response = self.client.get(url, content_type="application/json") self.assertTrue(200, response.status_code) + def test_base_resources_return_not_download_links_for_maps(self): + """ + Ensure we can access the Resource Base list. + """ + _map = Map.objects.first() + # From resource base API + url = reverse("base-resources-detail", args=[_map.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_urls", None) + self.assertListEqual([], download_url) + + # from maps api + url = reverse("maps-detail", args=[_map.id]) + download_url = response.json().get("resource").get("download_urls") + self.assertListEqual([], download_url) + + def test_base_resources_return_download_links_for_documents(self): + """ + Ensure we can access the Resource Base list. + """ + doc = Document.objects.first() + expected_payload = [{"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}] + # From resource base API + url = reverse("base-resources-detail", args=[doc.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_urls") + self.assertListEqual(expected_payload, download_url) + + # from documents api + url = reverse("documents-detail", args=[doc.id]) + download_url = response.json().get("resource").get("download_urls") + self.assertListEqual(expected_payload, download_url) + + def test_base_resources_return_download_links_for_datasets(self): + """ + Ensure we can access the Resource Base list. + """ + _dataset = Dataset.objects.first() + expected_payload = [ + {"url": reverse("dataset_download", args=[_dataset.alternate]), "ajax_safe": True, "default": True} + ] + + # From resource base API + url = reverse("base-resources-detail", args=[_dataset.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_urls") + self.assertEqual(expected_payload, download_url) + + # from dataset api + url = reverse("datasets-detail", args=[_dataset.id]) + download_url = response.json().get("resource").get("download_urls") + self.assertEqual(expected_payload, download_url) + class TestApiLinkedResources(GeoNodeBaseTestSupport): @classmethod diff --git a/geonode/documents/models.py b/geonode/documents/models.py index e9a81dac1dc..f482bd7abc4 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -103,6 +103,14 @@ def href(self): elif self.files: return urljoin(settings.SITEURL, reverse("document_link", args=(self.id,))) + @property + def is_local(self): + return False if self.doc_url else True + + @property + def download_is_ajax_safe(self): + return self.is_local + @property def is_file(self): return self.files and self.extension diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 369256492fa..1f80ac91afe 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -55,7 +55,7 @@ from geonode.documents.enumerations import DOCUMENT_TYPE_MAP from geonode.documents.models import Document, DocumentResourceLink -from geonode.base.populate_test_data import all_public, create_models, remove_models +from geonode.base.populate_test_data import all_public, create_models, create_single_doc, remove_models from geonode.upload.api.exceptions import FileUploadLimitException from .forms import DocumentCreateForm @@ -154,6 +154,26 @@ def test_create_document_url(self): self.assertEqual(doc.title, "GeoNode Map") self.assertEqual(doc.extension, "pdf") + def test_download_is_not_ajax_safe(self): + """Remote document is mark as not safe.""" + self.client.login(username="admin", password="admin") + form_data = { + "title": "A remote document through form is remote", + "doc_url": "https://development.demo.geonode.org/static/mapstore/img/geonode-logo.svg", + } + + response = self.client.post(reverse("document_upload"), data=form_data) + + self.assertEqual(response.status_code, 302) + + d = Document.objects.get(title="A remote document through form is remote") + self.assertFalse(d.download_is_ajax_safe) + + def test_download_is_ajax_safe(self): + """Remote document is mark as not safe.""" + d = create_single_doc("example_doc_name") + self.assertTrue(d.download_is_ajax_safe) + def test_create_document_url_view(self): """ Tests creating and updating external documents. diff --git a/geonode/geoserver/tests/test_server.py b/geonode/geoserver/tests/test_server.py index 1fbafbff07f..93a0c6971f4 100644 --- a/geonode/geoserver/tests/test_server.py +++ b/geonode/geoserver/tests/test_server.py @@ -48,6 +48,7 @@ from geonode.layers.populate_datasets_data import create_dataset_data from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset from geonode.geoserver.helpers import gs_catalog, get_sld_for, extract_name_from_sld +from geonode.catalogue.models import catalogue_post_save import logging @@ -1192,6 +1193,16 @@ def test_set_resources_links(self): with self.settings(UPDATE_RESOURCE_LINKS_AT_MIGRATE=True, ASYNC_SIGNALS=False): # Links _def_link_types = ["original", "metadata"] + Link.objects.update_or_create( + resource=Dataset.objects.first(), + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) _links = Link.objects.filter(link_type__in=_def_link_types) # Check 'original' and 'metadata' links exist self.assertIsNotNone(_links, "No 'original' and 'metadata' links have been found") @@ -1233,6 +1244,17 @@ def test_set_resources_links(self): for _lyr in _post_migrate_datasets: # Check original links in csw_anytext + # by default is not created anymore, we need to create one + Link.objects.update_or_create( + resource=_lyr, + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) _post_migrate_links_orig = Link.objects.filter( resource=_lyr.resourcebase_ptr, resource_id=_lyr.resourcebase_ptr.id, link_type="original" ) @@ -1240,6 +1262,9 @@ def test_set_resources_links(self): _post_migrate_links_orig.exists(), f"No 'original' links has been found for the layer '{_lyr.alternate}'", ) + # needed to update the csw_anytext field with the new link created + catalogue_post_save(instance=_lyr, sender=_lyr.__class__) + _lyr.refresh_from_db() for _link_orig in _post_migrate_links_orig: self.assertIn( _link_orig.url, diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index 7361e5c744a..88ba047ed03 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -30,6 +30,7 @@ from guardian.shortcuts import assign_perm, get_anonymous_user from geonode.geoserver.createlayer.utils import create_dataset +from geonode.base.models import Link from geonode.base.populate_test_data import create_models, create_single_dataset from geonode.layers.models import Attribute, Dataset from geonode.maps.models import Map, MapLayer @@ -415,3 +416,26 @@ def test_valid_metadata_file(self): put_data = {"metadata_file": f} response = self.client.put(url, data=put_data) self.assertEqual(200, response.status_code) + + def test_download_api(self): + dataset = create_single_dataset("test_dataset") + url = reverse("datasets-detail", kwargs={"pk": dataset.pk}) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + data = response.json()["dataset"] + download_url_data = data["download_urls"][0] + download_url = reverse("dataset_download", args=[dataset.alternate]) + self.assertEqual(download_url_data["default"], True) + self.assertEqual(download_url_data["ajax_safe"], True) + self.assertEqual(download_url_data["url"], download_url) + + link = Link(link_type="original", url="https://myoriginal.org", resource=dataset) + link.save() + + response = self.client.get(url) + data = response.json()["dataset"] + download_url_data = data["download_urls"][0] + download_url = reverse("dataset_download", args=[dataset.alternate]) + self.assertEqual(download_url_data["default"], True) + self.assertEqual(download_url_data["ajax_safe"], False) + self.assertEqual(download_url_data["url"], "https://myoriginal.org") diff --git a/geonode/layers/download_handler.py b/geonode/layers/download_handler.py new file mode 100644 index 00000000000..d969a0d3703 --- /dev/null +++ b/geonode/layers/download_handler.py @@ -0,0 +1,168 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging +import xml.etree.ElementTree as ET + +from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse +from django.template.loader import get_template +from django.urls import reverse +from django.utils.translation import ugettext as _ +from django.conf import settings +from geonode.base.auth import get_or_create_token +from geonode.geoserver.helpers import wps_format_is_supported +from geonode.layers.views import _resolve_dataset +from geonode.proxy.views import fetch_response_headers +from geonode.utils import HttpClient + +logger = logging.getLogger("geonode.layers.download_handler") + + +class DatasetDownloadHandler: + def __str__(self): + return f"{self.__module__}.{self.__class__.__name__}" + + def __repr__(self): + return self.__str__() + + def __init__(self, request, resource_name) -> None: + self.request = request + self.resource_name = resource_name + self._resource = None + + def get_download_response(self): + """ + Basic method. Should return the Response object + that allow the resource download + """ + resource = self.get_resource() + if not resource: + raise Http404("Resource requested is not available") + response = self.process_dowload(resource) + return response + + @property + def is_link_resource(self): + resource = self.get_resource() + return resource.link_set.filter(resource=resource, link_type="original").exists() + + @property + def is_ajax_safe(self): + """ + AJAX is safe to be used for WPS downloads. In case of a link set in a Link entry we cannot assume it, + since it could point to an external (non CORS enabled) URL + """ + return settings.USE_GEOSERVER and not self.is_link_resource + + @property + def download_url(self): + resource = self.get_resource() + if not resource: + return None + if resource.subtype not in ["vector", "raster", "vector_time"]: + logger.info("Download URL is available only for datasets that have been harvested and copied locally") + return None + + if self.is_link_resource: + return resource.link_set.filter(resource=resource.get_self_resource(), link_type="original").first().url + + return reverse("dataset_download", args=[resource.alternate]) + + def get_resource(self): + """ + Returnt the object needed + """ + if not self._resource: + try: + self._resource = _resolve_dataset( + self.request, + self.resource_name, + "base.download_resourcebase", + _("You do not have download permissions for this dataset."), + ) + except Exception as e: + logger.exception(e) + + return self._resource + + def process_dowload(self, resource=None): + """ + Generate the response object + """ + if not resource: + resource = self.get_resource() + if not settings.USE_GEOSERVER: + # if GeoServer is not used, we redirect to the proxy download + return HttpResponseRedirect(reverse("download", args=[resource.id])) + + download_format = self.request.GET.get("export_format") + + if download_format and not wps_format_is_supported(download_format, resource.subtype): + logger.error("The format provided is not valid for the selected resource") + return JsonResponse({"error": "The format provided is not valid for the selected resource"}, status=500) + + _format = "application/zip" if resource.is_vector() else "image/tiff" + # getting default payload + tpl = get_template("geoserver/dataset_download.xml") + ctx = {"alternate": resource.alternate, "download_format": download_format or _format} + # applying context for the payload + payload = tpl.render(ctx) + + # init of Client + client = HttpClient() + + headers = {"Content-type": "application/xml", "Accept": "application/xml"} + + # defining the URL needed fr the download + url = f"{settings.OGC_SERVER['default']['LOCATION']}ows?service=WPS&version=1.0.0&REQUEST=Execute" + if not self.request.user.is_anonymous: + # define access token for the user + access_token = get_or_create_token(self.request.user) + url += f"&access_token={access_token}" + + # request to geoserver + response, content = client.request(url=url, data=payload, method="post", headers=headers) + + if not response or response.status_code != 200: + logger.error(f"Download dataset exception: error during call with GeoServer: {content}") + return JsonResponse( + {"error": "Download dataset exception: error during call with GeoServer"}, + status=500, + ) + + # error handling + namespaces = {"ows": "http://www.opengis.net/ows/1.1", "wps": "http://www.opengis.net/wps/1.0.0"} + response_type = response.headers.get("Content-Type") + if response_type == "text/xml": + # parsing XML for get exception + content = ET.fromstring(response.text) + exc = content.find("*//ows:Exception", namespaces=namespaces) or content.find( + "ows:Exception", namespaces=namespaces + ) + if exc: + exc_text = exc.find("ows:ExceptionText", namespaces=namespaces) + logger.error(f"{exc.attrib.get('exceptionCode')} {exc_text.text}") + return JsonResponse({"error": f"{exc.attrib.get('exceptionCode')}: {exc_text.text}"}, status=500) + + return_response = fetch_response_headers( + HttpResponse(content=response.content, status=response.status_code, content_type=download_format), + response.headers, + ) + return_response.headers["Content-Type"] = download_format or _format + return return_response diff --git a/geonode/layers/models.py b/geonode/layers/models.py index 0d6c425e8e0..05df8956868 100644 --- a/geonode/layers/models.py +++ b/geonode/layers/models.py @@ -330,8 +330,6 @@ def download_url(self): if self.subtype not in ["vector", "raster", "vector_time"]: logger.info("Download URL is available only for datasets that have been harvested and copied locally") return None - if self.link_set.filter(resource=self.get_self_resource(), link_type="original").exists(): - return self.link_set.filter(resource=self.get_self_resource(), link_type="original").first().url return build_absolute_uri(reverse("dataset_download", args=(self.alternate,))) @property diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index d7420fc2a84..3899b1ad7ad 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -41,6 +41,7 @@ from django.contrib.gis.geos import Polygon from django.db.models import Count from django.contrib.auth import get_user_model +from django.http import HttpResponse from django.conf import settings from django.test.utils import override_settings @@ -49,6 +50,7 @@ from geonode.layers import utils from geonode.base import enumerations +from geonode.layers.utils import clear_dataset_download_handlers from geonode.layers import DatasetAppConfig from geonode.layers.admin import DatasetAdmin from geonode.decorators import on_ogc_backend @@ -81,6 +83,7 @@ ) from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset +from geonode.layers.download_handler import DatasetDownloadHandler logger = logging.getLogger(__name__) @@ -373,11 +376,6 @@ def test_dataset_links(self): links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="image") self.assertIsNotNone(links) - Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="original").update( - url="http://google.com/test" - ) - self.assertEqual(lyr.download_url, "http://google.com/test") - def test_get_valid_user(self): # Verify it accepts an admin user adminuser = get_user_model().objects.get(is_superuser=True) @@ -1201,6 +1199,7 @@ def test_dataset_download_redirect_to_proxy_url(self): self.assertEqual(302, response.status_code) self.assertEqual(f"/download/{dataset.id}", response.url) + @patch("geonode.layers.download_handler.HttpClient.request") def test_dataset_download_invalid_wps_format(self): # if settings.USE_GEOSERVER is false, the URL must be redirected self.client.login(username="admin", password="admin") @@ -1224,7 +1223,7 @@ def test_dataset_download_call_the_catalog_raise_error_for_no_200(self, mocked_c {"error": "Download dataset exception: error during call with GeoServer: foo-bar"}, response.json() ) - @patch("geonode.layers.views.HttpClient.request") + @patch("geonode.layers.download_handler.HttpClient.request") def test_dataset_download_call_the_catalog_raise_error_for_error_content(self, mocked_catalog): content = """ @@ -1249,7 +1248,7 @@ def test_dataset_download_call_the_catalog_works(self): self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.layers.views.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) @@ -1268,21 +1267,21 @@ def test_dataset_download_call_the_catalog_work_anonymous(self): _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.layers.views.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) self.assertTrue(response.status_code == 200) @override_settings(USE_GEOSERVER=True) - @patch("geonode.layers.views.get_template") + @patch("geonode.layers.download_handler.get_template") def test_dataset_download_call_the_catalog_work_for_raster(self, pathed_template): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.filter(subtype="raster").first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") Dataset.objects.filter(alternate=layer.alternate).update(subtype="raster") - with patch("geonode.layers.views.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) @@ -1295,13 +1294,13 @@ def test_dataset_download_call_the_catalog_work_for_raster(self, pathed_template ) @override_settings(USE_GEOSERVER=True) - @patch("geonode.layers.views.get_template") + @patch("geonode.layers.download_handler.get_template") def test_dataset_download_call_the_catalog_work_for_vector(self, pathed_template): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.filter(subtype="vector").first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.layers.views.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) @@ -2221,3 +2220,54 @@ def _assert_perms(self, expected_perms, dataset, username, assertion=True): self.assertSetEqual(expected_perms, actual) else: self.assertFalse(username in [user.username for user in perms["users"]]) + + +class TestDatasetDownloadHandler(GeoNodeBaseTestSupport): + def setUp(self): + user = get_user_model().objects.first() + request = RequestFactory().get("http://test_url.com") + request.user = user + self.dataset = create_single_dataset("test_dataset_for_download") + self.sut = DatasetDownloadHandler(request, self.dataset.alternate) + + def test_download_url_without_original_link(self): + expected_url = reverse("dataset_download", args=[self.dataset.alternate]) + self.assertEqual(expected_url, self.sut.download_url) + + def test_download_url_with_original_link(self): + Link.objects.update_or_create( + resource=self.dataset.resourcebase_ptr, + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + expected_url = "https://custom_dowonload_url.com" + self.assertEqual(expected_url, self.sut.download_url) + + def test_get_resource_exists(self): + self.assertIsNotNone(self.sut.get_resource()) + + def test_process_dowload(self): + response = self.sut.get_download_response() + self.assertIsNotNone(response) + + +class DummyDownloadHandler(DatasetDownloadHandler): + def get_download_response(self): + return HttpResponse(content=b"abcsfd2") + + +class TestCustomDownloadHandler(GeoNodeBaseTestSupport): + @override_settings(DEFAULT_DATASET_DOWNLOAD_HANDLER="geonode.layers.tests.DummyDownloadHandler") + def test_download_custom_handler(self): + clear_dataset_download_handlers() + dataset = create_single_dataset("test_custom_download_dataset") + url = reverse("dataset_download", args=[dataset.alternate]) + self.client.login(username="admin", password="admin") + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + self.assertEqual(response.content, b"abcsfd2") diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py index 09ef6f02b76..c5515e1d267 100644 --- a/geonode/layers/utils.py +++ b/geonode/layers/utils.py @@ -44,6 +44,7 @@ from geonode.layers.api.exceptions import InvalidDatasetException from geonode.security.permissions import PermSpec, PermSpecCompact from geonode.storage.manager import storage_manager +from django.utils.module_loading import import_string # Geonode functionality from geonode.base.models import Region @@ -591,3 +592,37 @@ def is_sld_upload_only(request): def mdata_search_by_type(request, filetype): files = list({v.name for k, v in request.FILES.items()}) return len(files) == 1 and all([filetype in f for f in files]) + + +default_dataset_download_handler = None +dataset_download_handler_list = [] + + +def get_dataset_download_handlers(): + if not dataset_download_handler_list and getattr(settings, "DATASET_DOWNLOAD_HANDLERS", None): + dataset_download_handler_list.append(import_string(settings.DATASET_DOWNLOAD_HANDLERS[0])) + + return dataset_download_handler_list + + +def get_default_dataset_download_handler(): + global default_dataset_download_handler + if not default_dataset_download_handler and getattr(settings, "DEFAULT_DATASET_DOWNLOAD_HANDLER", None): + default_dataset_download_handler = import_string(settings.DEFAULT_DATASET_DOWNLOAD_HANDLER) + + return default_dataset_download_handler + + +def set_default_dataset_download_handler(handler): + global default_dataset_download_handler + handler_module = import_string(handler) + if handler_module not in dataset_download_handler_list: + dataset_download_handler_list.append(handler_module) + + default_dataset_download_handler = handler_module + + +def clear_dataset_download_handlers(): + global default_dataset_download_handler + dataset_download_handler_list.clear() + default_dataset_download_handler = None diff --git a/geonode/layers/views.py b/geonode/layers/views.py index 609a7f0c839..3aa3bb91ff8 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -23,21 +23,18 @@ import logging import warnings import traceback -from django.urls import reverse from owslib.wfs import WebFeatureService -import xml.etree.ElementTree as ET from django.conf import settings from django.db.models import F -from django.http import Http404, JsonResponse +from django.http import Http404 from django.contrib import messages from django.shortcuts import render from django.utils.html import escape from django.forms.utils import ErrorList from django.contrib.auth import get_user_model -from django.template.loader import get_template from django.utils.translation import ugettext as _ from django.core.exceptions import PermissionDenied from django.forms.models import inlineformset_factory @@ -50,9 +47,8 @@ from geonode import geoserver from geonode.layers.metadata import parse_metadata -from geonode.proxy.views import fetch_response_headers from geonode.resource.manager import resource_manager -from geonode.geoserver.helpers import set_dataset_style, wps_format_is_supported +from geonode.geoserver.helpers import set_dataset_style from geonode.resource.utils import update_resource from geonode.base.auth import get_or_create_token @@ -63,14 +59,19 @@ from geonode.decorators import check_keyword_write_perms from geonode.layers.forms import DatasetForm, DatasetTimeSerieForm, LayerAttributeForm, NewLayerUploadForm from geonode.layers.models import Dataset, Attribute -from geonode.layers.utils import is_sld_upload_only, is_xml_upload_only, validate_input_source +from geonode.layers.utils import ( + is_sld_upload_only, + is_xml_upload_only, + get_default_dataset_download_handler, + validate_input_source, +) from geonode.services.models import Service from geonode.base import register_event from geonode.monitoring.models import EventType from geonode.groups.models import GroupProfile from geonode.security.utils import get_user_visible_groups, AdvancedSecurityWorkflowManager from geonode.people.forms import ProfileForm -from geonode.utils import HttpClient, check_ogc_backend, llbbox_to_mercator, resolve_object, mkdtemp +from geonode.utils import check_ogc_backend, llbbox_to_mercator, resolve_object, mkdtemp from geonode.geoserver.helpers import ogc_server_settings, select_relevant_files, write_uploaded_files_to_disk from geonode.geoserver.security import set_geowebcache_invalidate_cache @@ -736,69 +737,8 @@ def dataset_metadata_advanced(request, layername): @csrf_exempt def dataset_download(request, layername): - try: - dataset = _resolve_dataset(request, layername, "base.download_resourcebase", _PERMISSION_MSG_GENERIC) - except Exception as e: - raise Http404(Exception(_("Not found"), e)) - - if not settings.USE_GEOSERVER: - # if GeoServer is not used, we redirect to the proxy download - return HttpResponseRedirect(reverse("download", args=[dataset.id])) - - download_format = request.GET.get("export_format") - - if download_format and not wps_format_is_supported(download_format, dataset.subtype): - logger.error("The format provided is not valid for the selected resource") - return JsonResponse({"error": "The format provided is not valid for the selected resource"}, status=500) - - _format = "application/zip" if dataset.is_vector() else "image/tiff" - # getting default payload - tpl = get_template("geoserver/dataset_download.xml") - ctx = {"alternate": dataset.alternate, "download_format": download_format or _format} - # applying context for the payload - payload = tpl.render(ctx) - - # init of Client - client = HttpClient() - - headers = {"Content-type": "application/xml", "Accept": "application/xml"} - - # defining the URL needed fr the download - url = f"{settings.OGC_SERVER['default']['LOCATION']}ows?service=WPS&version=1.0.0&REQUEST=Execute" - if not request.user.is_anonymous: - # define access token for the user - access_token = get_or_create_token(request.user) - url += f"&access_token={access_token}" - - # request to geoserver - response, content = client.request(url=url, data=payload, method="post", headers=headers) - - if response.status_code != 200: - logger.error(f"Download dataset exception: error during call with GeoServer: {response.content}") - return JsonResponse( - {"error": f"Download dataset exception: error during call with GeoServer: {response.content}"}, status=500 - ) - - # error handling - namespaces = {"ows": "http://www.opengis.net/ows/1.1", "wps": "http://www.opengis.net/wps/1.0.0"} - response_type = response.headers.get("Content-Type") - if response_type == "text/xml": - # parsing XML for get exception - content = ET.fromstring(response.text) - exc = content.find("*//ows:Exception", namespaces=namespaces) or content.find( - "ows:Exception", namespaces=namespaces - ) - if exc: - exc_text = exc.find("ows:ExceptionText", namespaces=namespaces) - logger.error(f"{exc.attrib.get('exceptionCode')} {exc_text.text}") - return JsonResponse({"error": f"{exc.attrib.get('exceptionCode')}: {exc_text.text}"}, status=500) - - return_response = fetch_response_headers( - HttpResponse(content=response.content, status=response.status_code, content_type=download_format), - response.headers, - ) - return_response.headers["Content-Type"] = download_format or _format - return return_response + handler = get_default_dataset_download_handler() + return handler(request, layername).get_download_response() @login_required diff --git a/geonode/security/permissions.py b/geonode/security/permissions.py index 0c3ddac2147..4d8b2e0bf1f 100644 --- a/geonode/security/permissions.py +++ b/geonode/security/permissions.py @@ -571,9 +571,9 @@ def compact(self): } ) - json["users"] = user_perms - json["organizations"] = organization_perms - json["groups"] = group_perms + json["users"] = sorted(user_perms, key=lambda x: x["id"], reverse=True) + json["organizations"] = sorted(organization_perms, key=lambda x: x["id"], reverse=True) + json["groups"] = sorted(group_perms, key=lambda x: x["id"], reverse=True) return json.copy() diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 52b43a38551..28362a0d6f5 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -2480,8 +2480,9 @@ def test_user_with_view_perms(self): # setting the view permissions url = reverse(_case["url"], kwargs={"pk": _case["resource"].pk}) - _case["resource"].set_permissions({"users": {self.marty.username: ["base.view_resourcebase"]}}) - # calling the api + _case["resource"].set_permissions( + {"users": {self.marty.username: ["base.view_resourcebase", "base.download_resourcebase"]}} + ) # calling the api self.client.force_login(self.marty) result = self.client.get(url) # checking that the user can call the url in get diff --git a/geonode/settings.py b/geonode/settings.py index d408093575b..dfe4e70c0b8 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -1340,8 +1340,6 @@ ] -DISPLAY_ORIGINAL_DATASET_LINK = ast.literal_eval(os.getenv("DISPLAY_ORIGINAL_DATASET_LINK", "True")) - ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE = ast.literal_eval(os.getenv("ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE", "False")) TASTYPIE_DEFAULT_FORMATS = ["json"] @@ -2325,3 +2323,7 @@ def get_geonode_catalogue_service(): "geonode.facets.providers.thesaurus.ThesaurusFacetProvider", "geonode.facets.providers.region.RegionFacetProvider", ) + +DEFAULT_DATASET_DOWNLOAD_HANDLER = "geonode.layers.download_handler.DatasetDownloadHandler" + +DATASET_DOWNLOAD_HANDLERS = ast.literal_eval(os.getenv("DATASET_DOWNLOAD_HANDLERS", "[]")) diff --git a/geonode/utils.py b/geonode/utils.py index 60a501d863a..e030758fc68 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -1377,10 +1377,7 @@ def get_legend_url( def set_resource_default_links(instance, layer, prune=False, **kwargs): from geonode.base.models import Link - from django.urls import reverse from django.utils.translation import ugettext - from geonode.layers.models import Dataset - from geonode.documents.models import Document # Prune old links if prune: @@ -1447,32 +1444,6 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs): logger.exception(e) bbox = instance.bbox_string - # Create Raw Data download link - if settings.DISPLAY_ORIGINAL_DATASET_LINK: - logger.debug(" -- Resource Links[Create Raw Data download link]...") - if isinstance(instance, Dataset): - download_url = build_absolute_uri(reverse("dataset_download", args=(instance.alternate,))) - elif isinstance(instance, Document): - download_url = build_absolute_uri(reverse("document_download", args=(instance.id,))) - else: - download_url = None - - while Link.objects.filter(resource=instance.resourcebase_ptr, link_type="original").exists(): - Link.objects.filter(resource=instance.resourcebase_ptr, link_type="original").delete() - Link.objects.update_or_create( - resource=instance.resourcebase_ptr, - url=download_url, - defaults=dict( - extension="zip", - name="Original Dataset", - mime="application/octet-stream", - link_type="original", - ), - ) - logger.debug(" -- Resource Links[Create Raw Data download link]...done!") - else: - Link.objects.filter(resource=instance.resourcebase_ptr, name="Original Dataset").delete() - # Set download links for WMS, WCS or WFS and KML logger.debug(" -- Resource Links[Set download links for WMS, WCS or WFS and KML]...") instance_ows_url = f"{instance.ows_url}?" if instance.ows_url else f"{ogc_server_settings.public_url}ows?" From 99a1a1b4775f9ff849e571a95ca11343cb70203c Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Mon, 16 Oct 2023 14:41:17 +0200 Subject: [PATCH 2/2] Fix circleci test --- geonode/base/api/tests.py | 33 ++++++++------- geonode/layers/tests.py | 3 +- geonode/upload/api/tests.py | 80 ++++++++++++++++++++++++++----------- 3 files changed, 76 insertions(+), 40 deletions(-) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 4805cf7b12b..4254cf6be90 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -2013,6 +2013,7 @@ def test_manager_can_edit_map(self): """ REST API must not forbid saving maps and apps to non-admin and non-owners. """ + self.maxDiff = None from geonode.maps.models import Map _map = Map.objects.filter(uuid__isnull=False, owner__username="admin").first() @@ -2068,7 +2069,7 @@ def test_manager_can_edit_map(self): response = self.client.get(resource_service_permissions_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data - self.assertEqual( + self.assertDictEqual( resource_perm_spec, { "users": [ @@ -2109,7 +2110,7 @@ def test_manager_can_edit_map(self): response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data - self.assertEqual( + self.assertDictEqual( resource_perm_spec, { "users": [ @@ -2148,20 +2149,10 @@ def test_manager_can_edit_map(self): response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data - self.assertEqual( + self.assertDictEqual( resource_perm_spec, { "users": [ - { - "id": 1, - "username": "admin", - "first_name": "admin", - "last_name": "", - "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", - "permissions": "owner", - "is_staff": True, - "is_superuser": True, - }, { "id": bobby.id, "username": "bobby", @@ -2172,6 +2163,16 @@ def test_manager_can_edit_map(self): "is_staff": False, "is_superuser": False, }, + { + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "permissions": "owner", + "is_staff": True, + "is_superuser": True, + }, ], "organizations": [], "groups": [ @@ -2468,7 +2469,7 @@ def test_base_resources_return_not_download_links_for_maps(self): """ Ensure we can access the Resource Base list. """ - _map = Map.objects.first() + _map = create_single_map("map_1") # From resource base API url = reverse("base-resources-detail", args=[_map.id]) response = self.client.get(url, format="json") @@ -2479,12 +2480,13 @@ def test_base_resources_return_not_download_links_for_maps(self): url = reverse("maps-detail", args=[_map.id]) download_url = response.json().get("resource").get("download_urls") self.assertListEqual([], download_url) + _map.delete() def test_base_resources_return_download_links_for_documents(self): """ Ensure we can access the Resource Base list. """ - doc = Document.objects.first() + doc = create_single_doc("doc_1") expected_payload = [{"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}] # From resource base API url = reverse("base-resources-detail", args=[doc.id]) @@ -2496,6 +2498,7 @@ def test_base_resources_return_download_links_for_documents(self): url = reverse("documents-detail", args=[doc.id]) download_url = response.json().get("resource").get("download_urls") self.assertListEqual(expected_payload, download_url) + doc.delete() def test_base_resources_return_download_links_for_datasets(self): """ diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 3899b1ad7ad..1d7e1285a0e 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -1199,7 +1199,6 @@ def test_dataset_download_redirect_to_proxy_url(self): self.assertEqual(302, response.status_code) self.assertEqual(f"/download/{dataset.id}", response.url) - @patch("geonode.layers.download_handler.HttpClient.request") def test_dataset_download_invalid_wps_format(self): # if settings.USE_GEOSERVER is false, the URL must be redirected self.client.login(username="admin", password="admin") @@ -1209,7 +1208,7 @@ def test_dataset_download_invalid_wps_format(self): self.assertEqual(500, response.status_code) self.assertDictEqual({"error": "The format provided is not valid for the selected resource"}, response.json()) - @patch("geonode.layers.views.HttpClient.request") + @patch("geonode.layers.download_handler.HttpClient.request") def test_dataset_download_call_the_catalog_raise_error_for_no_200(self, mocked_catalog): _response = MagicMock(status_code=500, content="foo-bar") mocked_catalog.return_value = _response, "foo-bar" diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py index 8f84fec6a92..81cd4a3ea3f 100644 --- a/geonode/upload/api/tests.py +++ b/geonode/upload/api/tests.py @@ -17,7 +17,9 @@ # ######################################################################### +from geonode.base.models import ResourceBase from geonode.resource.models import ExecutionRequest +from geonode.geoserver.helpers import gs_catalog import os import shutil import logging @@ -233,35 +235,67 @@ def test_rest_uploads(self): """ Ensure we can access the Local Server Uploads list. """ - # Try to upload a good raster file and check the session IDs - fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") - resp, data = rest_upload_by_path(fname, self.client) - self.assertEqual(resp.status_code, 201) - - url = reverse("uploads-list") - # Anonymous - self.client.logout() - response = self.client.get(url, format="json") - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 5) - self.assertEqual(response.data["total"], 0) - # Pagination - self.assertEqual(len(response.data["uploads"]), 0) - logger.debug(response.data) + resp = None + layer_name = "relief_san_andres" + try: + self._cleanup_layer(layer_name=layer_name) + # Try to upload a good raster file and check the session IDs + fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") + resp, data = rest_upload_by_path(fname, self.client) + self.assertEqual(resp.status_code, 201) + + url = reverse("uploads-list") + # Anonymous + self.client.logout() + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 0) + # Pagination + self.assertEqual(len(response.data["uploads"]), 0) + logger.debug(response.data) + except Exception: + if resp.json().get("errors"): + layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + finally: + self._cleanup_layer(layer_name) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_rest_uploads_non_interactive(self): """ Ensure we can access the Local Server Uploads list. """ - # Try to upload a good raster file and check the session IDs - fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") - resp, data = rest_upload_by_path(fname, self.client, non_interactive=True) - self.assertEqual(resp.status_code, 201) - - exec_id = data.get("execution_id", None) - _exec = ExecutionRequest.objects.get(exec_id=exec_id) - self.assertEqual(_exec.status, "finished") + resp = None + layer_name = "relief_san_andres" + try: + self._cleanup_layer(layer_name=layer_name) + # Try to upload a good raster file and check the session IDs + fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") + resp, data = rest_upload_by_path(fname, self.client, non_interactive=True) + self.assertEqual(resp.status_code, 201) + exec_id = data.get("execution_id", None) + _exec = ExecutionRequest.objects.get(exec_id=exec_id) + self.assertEqual(_exec.status, "finished") + except Exception: + if resp.json().get("errors"): + layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + finally: + self._cleanup_layer(layer_name) + + def _cleanup_layer(self, layer_name): + # removing the layer from geonode + x = ResourceBase.objects.filter(alternate__icontains=layer_name) + if x.exists(): + for el in x.iterator(): + el.delete() + # removing the layer from geoserver + dataset = gs_catalog.get_layer(layer_name) + if dataset: + gs_catalog.delete(dataset, purge="all", recurse=True) + # removing the layer from geoserver + store = gs_catalog.get_store(layer_name, workspace="geonode") + if store: + gs_catalog.delete(store, purge="all", recurse=True) @mock.patch("geonode.upload.uploadhandler.SimpleUploadedFile") def test_rest_uploads_with_size_limit(self, mocked_uploaded_file):